验证码验证不区分访问路径

build/build.sh

@@ -6,8 +6,11 @@ function build() {
 	VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
 	DIST=$ROOT/"../dist/${NAME}"
 	MUSL_DIR="/usr/local/opt/musl-cross/bin"
-	GCC_X86_64_DIR="/usr/local/Cellar/x86_64-unknown-linux-gnu/10.3.0/bin"
-	GCC_ARM64_DIR="/usr/local/Cellar/aarch64-unknown-linux-gnu/10.3.0/bin"
+	SRCDIR=$(realpath "$ROOT/..")
+
+	# for macOS users: precompiled gcc can be downloaded from https://github.com/messense/homebrew-macos-cross-toolchains
+	GCC_X86_64_DIR="/usr/local/gcc/x86_64-unknown-linux-gnu/bin"
+	GCC_ARM64_DIR="//usr/local/gcc/aarch64-unknown-linux-gnu/bin"
 
 	OS=${1}
 	ARCH=${2}
@@ -57,7 +60,10 @@ function build() {
 	# we support TOA on linux only
 	if [ "$OS" == "linux" ] && [ -f "${ROOT}/edge-toa/edge-toa-${ARCH}" ]
 	then
-		mkdir "$DIST/edge-toa"
+		if [ ! -d  "$DIST/edge-toa" ]
+		then
+			mkdir "$DIST/edge-toa"
+		fi
 		cp "${ROOT}/edge-toa/edge-toa-${ARCH}" "$DIST/edge-toa/edge-toa"
 	fi
 
@@ -65,6 +71,8 @@ function build() {
 
 	CC_PATH=""
 	CXX_PATH=""
+	CGO_LDFLAGS=""
+	CGO_CFLAGS=""
 	BUILD_TAG=$TAG
 	if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
 		if [ "${ARCH}" == "amd64" ]; then
@@ -74,7 +82,7 @@ function build() {
 				CC_PATH="x86_64-unknown-linux-gnu-gcc"
 				CXX_PATH="x86_64-unknown-linux-gnu-g++"
 				if [ "$TAG" = "plus" ]; then
-					BUILD_TAG="plus,script"
+					BUILD_TAG="plus,script,packet"
 				fi
 			else
 				CC_PATH="x86_64-linux-musl-gcc"
@@ -92,7 +100,7 @@ function build() {
 				CC_PATH="aarch64-unknown-linux-gnu-gcc"
 				CXX_PATH="aarch64-unknown-linux-gnu-g++"
 				if [ "$TAG" = "plus" ]; then
-					BUILD_TAG="plus,script"
+					BUILD_TAG="plus,script,packet"
 				fi
 			else
 				CC_PATH="aarch64-linux-musl-gcc"
@@ -112,13 +120,31 @@ function build() {
 			CXX_PATH="mips64el-linux-musl-g++"
 		fi
 	fi
+
+	# libpcap
+	if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] &&  [ "$TAG" == "plus" ]; then
+		CGO_LDFLAGS="-L${SRCDIR}/libs/libpcap/${ARCH} -lpcap"
+		CGO_CFLAGS="-I${SRCDIR}/libs/libpcap/src/libpcap -I${SRCDIR}/libs/libpcap/src/libpcap/pcap"
+	fi
+
 	if [ ! -z $CC_PATH ]; then
-		env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
+		env CC=$MUSL_DIR/$CC_PATH \
+		 CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" \
+		 GOARCH="${ARCH}" CGO_ENABLED=1 \
+		 CGO_LDFLAGS="${CGO_LDFLAGS}" \
+		 CGO_CFLAGS="${CGO_CFLAGS}" \
+		 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
 	else
 		if [[ `uname` == *"Linux"* ]] && [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] &&  [ "$TAG" == "plus" ]; then
-			BUILD_TAG="plus,script"
+			BUILD_TAG="plus,script,packet"
 		fi
-		env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
+
+		env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1  CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
+	fi
+
+	if [ ! -f "${DIST}/bin/${NAME}" ]; then
+		echo "build failed!"
+		exit
 	fi
 
 	# delete hidden files

cmd/edge-node/main.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -30,7 +31,7 @@ func main() {
 		Version(teaconst.Version).
 		Product(teaconst.ProductName).
 		Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog|uninstall]").
-		Usage(teaconst.ProcessName + " [trackers|goman|conns|gc|bandwidth|disk]").
+		Usage(teaconst.ProcessName + " [trackers|goman|conns|gc|bandwidth|disk|cache.garbage]").
 		Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove|ip.close] IP")
 
 	app.On("start:before", func() {
@@ -464,6 +465,58 @@ func main() {
 			fmt.Println("Usage: edge-node disk [speed]")
 		}
 	})
+	app.On("cache.garbage", func() {
+		fmt.Println("scanning ...")
+
+		var shouldDelete bool
+		for _, arg := range os.Args {
+			if strings.TrimLeft(arg, "-") == "delete" {
+				shouldDelete = true
+			}
+		}
+
+		var progressSock = gosock.NewTmpSock(teaconst.CacheGarbageSockName)
+		progressSock.OnCommand(func(cmd *gosock.Command) {
+			var params = maps.NewMap(cmd.Params)
+			if cmd.Code == "progress" {
+				fmt.Printf("%.2f%% %d\n", params.GetFloat64("progress")*100, params.GetInt("count"))
+				_ = cmd.ReplyOk()
+			}
+		})
+		go func() {
+			_ = progressSock.Listen()
+		}()
+		time.Sleep(1 * time.Second)
+
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{
+			Code:   "cache.garbage",
+			Params: map[string]any{"delete": shouldDelete},
+		})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+			return
+		}
+
+		var params = maps.NewMap(reply.Params)
+		if params.GetBool("isOk") {
+			var count = params.GetInt("count")
+			fmt.Println("found", count, "bad caches")
+
+			if count > 0 {
+				fmt.Println("======")
+				var sampleFiles = params.GetSlice("sampleFiles")
+				for _, file := range sampleFiles {
+					fmt.Println(types.String(file))
+				}
+				if count > len(sampleFiles) {
+					fmt.Println("... more files")
+				}
+			}
+		} else {
+			fmt.Println("[ERROR]" + params.GetString("error"))
+		}
+	})
 	app.Run(func() {
 		var node = nodes.NewNode()
 		node.Start()

go.mod

@@ -13,46 +13,47 @@ require (
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible
 	github.com/andybalholm/brotli v1.0.5
 	github.com/aws/aws-sdk-go v1.44.279
+	github.com/baidubce/bce-sdk-go v0.9.153
 	github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670
 	github.com/cespare/xxhash v1.1.0
 	github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/golang/protobuf v1.5.3
+	github.com/google/gopacket v1.1.19
 	github.com/google/nftables v0.1.0
 	github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible
 	github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d
 	github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
 	github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
-	github.com/iwind/gowebp v0.0.0-20230615040911-5013dbb9d508
-	github.com/klauspost/compress v1.16.5
-	github.com/mattn/go-sqlite3 v1.14.9
+	github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4
+	github.com/klauspost/compress v1.17.2
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mdlayher/netlink v1.7.1
 	github.com/miekg/dns v1.1.43
 	github.com/mssola/useragent v1.0.0
 	github.com/pires/go-proxyproto v0.6.1
 	github.com/qiniu/go-sdk/v7 v7.16.0
-	github.com/quic-go/quic-go v0.36.1
+	github.com/quic-go/quic-go v0.39.2
 	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.41
-	golang.org/x/image v0.7.0
-	golang.org/x/net v0.12.0
-	golang.org/x/sys v0.10.0
-	google.golang.org/grpc v1.55.0
+	golang.org/x/image v0.13.0
+	golang.org/x/net v0.17.0
+	golang.org/x/sys v0.13.0
+	google.golang.org/grpc v1.59.0
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chai2010/webp v1.1.1 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
-	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
+	github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.0.0 // indirect
 	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
@@ -60,23 +61,22 @@ require (
 	github.com/mdlayher/socket v0.4.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mozillazg/go-httpheader v0.2.1 // indirect
-	github.com/onsi/ginkgo/v2 v2.11.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.13.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
-	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
+	github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
 	github.com/tdewolff/minify/v2 v2.12.7 // indirect
 	github.com/tdewolff/parse/v2 v2.6.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
 	github.com/tklauser/numcpus v0.3.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	go.uber.org/mock v0.3.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.11.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 )

go.sum

@@ -7,14 +7,14 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/aws/aws-sdk-go v1.44.279 h1:g23dxnYjIiPlQo0gIKNR0zVPsSvo1bj5frWln+5sfhk=
 github.com/aws/aws-sdk-go v1.44.279/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/baidubce/bce-sdk-go v0.9.153 h1:h5l2EXehe4C4/bdlAPBaULrbnEDgIu5HOYgniN7bjGM=
+github.com/baidubce/bce-sdk-go v0.9.153/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
 github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
 github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
-github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
 github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
@@ -41,8 +41,6 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
-github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -53,10 +51,12 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA=
-github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0=
+github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible h1:XRAk4HBDLCYEdPLWtKf5iZhOi7lfx17aY0oSO9+mcg8=
 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
 github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d h1:XnTIj781NdSipts60fVqbgZorVAKVSRaA6nqVNfBQ1g=
@@ -67,8 +67,8 @@ github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedV
 github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
 github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
 github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
-github.com/iwind/gowebp v0.0.0-20230615040911-5013dbb9d508 h1:fjKiHAyPQmdwuw1DQ2BI1JTbhUWAtI3Kr9wIZQBdRgQ=
-github.com/iwind/gowebp v0.0.0-20230615040911-5013dbb9d508/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
+github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4 h1:eyymORsZg0tZ0niyolYF4nao4sdNUI+Ll40s96tKHBY=
+github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
 github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4 h1:RPAH9Sj9l/20zH5zU5/iJGszfwPq6eLjoiC/n/asulA=
 github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4/go.mod h1:7OLL+86wZKfBnAJxNxmdcZ0ebbgdp/A28fcagx9oJqA=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -79,8 +79,8 @@ github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTx
 github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -93,8 +93,8 @@ github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ic
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
-github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
-github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mdlayher/netlink v1.7.1 h1:FdUaT/e33HjEXagwELR8R3/KL1Fq5x3G5jgHLp/BTmg=
 github.com/mdlayher/netlink v1.7.1/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ=
 github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=
@@ -109,9 +109,9 @@ github.com/mssola/useragent v1.0.0 h1:WRlDpXyxHDNfvZaPEut5Biveq86Ze4o4EMffyMxmH5
 github.com/mssola/useragent v1.0.0/go.mod h1:hz9Cqz4RXusgg1EdI4Al0INR62kP7aPSRNHnpU+b85Y=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
-github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
-github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
 github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -126,12 +126,10 @@ github.com/qiniu/go-sdk/v7 v7.16.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYX
 github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
-github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
-github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E=
-github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
-github.com/quic-go/quic-go v0.36.1 h1:WsG73nVtnDy1TiACxFxhQ3TqaW+DipmqzLEtNlAwZyY=
-github.com/quic-go/quic-go v0.36.1/go.mod h1:zPetvwDlILVxt15n3hr3Gf/I3mDf7LpLKPhR4Ez0AZQ=
+github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
+github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
+github.com/quic-go/quic-go v0.39.2 h1:hmwAf8zAHlvan0Y5PXxeeBFZEW17IW99sXLry8I2kjk=
+github.com/quic-go/quic-go v0.39.2/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
@@ -161,49 +159,45 @@ github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ
 github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
 github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
+go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
-golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
-golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
-golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
+golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -212,45 +206,39 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
-golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
-google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/caches/item.go

@@ -3,7 +3,6 @@ package caches
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"strings"
-	"time"
 )
 
 type ItemType = int
@@ -16,7 +15,7 @@ const (
 // 计算当前周
 // 不要用YW，因为需要计算两周是否临近
 func currentWeek() int32 {
-	return int32(time.Now().Unix() / 86400)
+	return int32(fasttime.Now().Unix() / 86400)
 }
 
 type Item struct {
@@ -29,10 +28,7 @@ type Item struct {
 	MetaSize   int64    `json:"metaSize"`
 	Host       string   `json:"host"`     // 主机名
 	ServerId   int64    `json:"serverId"` // 服务ID
-
-	Week1Hits int64 `json:"week1Hits"`
-	Week2Hits int64 `json:"week2Hits"`
-	Week      int32 `json:"week"`
+	Week       int32    `json:"week"`
 }
 
 func (this *Item) IsExpired() bool {
@@ -47,20 +43,6 @@ func (this *Item) Size() int64 {
 	return this.HeaderSize + this.BodySize
 }
 
-func (this *Item) IncreaseHit(week int32) {
-	if this.Week == week {
-		this.Week2Hits++
-	} else {
-		if week-this.Week == 1 {
-			this.Week1Hits = this.Week2Hits
-		} else {
-			this.Week1Hits = 0
-		}
-		this.Week2Hits = 1
-		this.Week = week
-	}
-}
-
 func (this *Item) RequestURI() string {
 	var schemeIndex = strings.Index(this.Key, "://")
 	if schemeIndex <= 0 {

internal/caches/item_test.go

@@ -1,8 +1,9 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package caches
+package caches_test
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
@@ -11,27 +12,14 @@ import (
 	"time"
 )
 
-func TestItem_IncreaseHit(t *testing.T) {
-	var week = currentWeek()
-
-	var item = &Item{}
-	//item.Week = 2704
-	item.Week2Hits = 100
-	item.IncreaseHit(week)
-	t.Log("week:", item.Week, "week1:", item.Week1Hits, "week2:", item.Week2Hits)
-
-	item.IncreaseHit(week)
-	t.Log("week:", item.Week, "week1:", item.Week1Hits, "week2:", item.Week2Hits)
-}
-
 func TestItems_Memory(t *testing.T) {
 	var stat = &runtime.MemStats{}
 	runtime.ReadMemStats(stat)
 	var memory1 = stat.HeapInuse
 
-	var items = []*Item{}
+	var items = []*caches.Item{}
 	for i := 0; i < 10_000_000; i++ {
-		items = append(items, &Item{
+		items = append(items, &caches.Item{
 			Key: types.String(i),
 		})
 	}
@@ -41,18 +29,11 @@ func TestItems_Memory(t *testing.T) {
 
 	t.Log(memory1, memory2, (memory2-memory1)/1024/1024, "M")
 
-	var weekItems = make(map[string]*Item, 10_000_000)
-
-	for _, item := range items {
-		weekItems[item.Key] = item
-	}
-
 	runtime.ReadMemStats(stat)
 	var memory3 = stat.HeapInuse
 	t.Log(memory2, memory3, (memory3-memory2)/1024/1024, "M")
 
 	time.Sleep(1 * time.Second)
-	t.Log(len(items), len(weekItems))
 }
 
 func TestItems_Memory2(t *testing.T) {
@@ -88,7 +69,7 @@ func TestItem_RequestURI(t *testing.T) {
 		"https://goedge.cn:8080/hello/world",
 		"https://goedge.cn/hello/world?v=1&t=123",
 	} {
-		var item = &Item{Key: u}
+		var item = &caches.Item{Key: u}
 		t.Log(u, "=>", item.RequestURI())
 	}
 }

internal/caches/list_file.go

@@ -4,13 +4,18 @@ package caches
 
 import (
 	"database/sql"
+	"errors"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/types"
 	"os"
+	"strings"
+	"sync"
 	"time"
 )
 
@@ -24,7 +29,7 @@ type FileList struct {
 	onAdd    func(item *Item)
 	onRemove func(item *Item)
 
-	memoryCache *ttlcache.Cache
+	memoryCache *ttlcache.Cache[zero.Zero]
 
 	// 老数据库地址
 	oldDir string
@@ -33,7 +38,7 @@ type FileList struct {
 func NewFileList(dir string) ListInterface {
 	return &FileList{
 		dir:         dir,
-		memoryCache: ttlcache.NewCache(),
+		memoryCache: ttlcache.NewCache[zero.Zero](),
 	}
 }
 
@@ -59,19 +64,37 @@ func (this *FileList) Init() error {
 	}
 
 	remotelogs.Println("CACHE", "loading database from '"+dir+"' ...")
+	var wg = &sync.WaitGroup{}
+	var locker = sync.Mutex{}
+	var lastErr error
+
 	for i := 0; i < CountFileDB; i++ {
-		var db = NewFileListDB()
-		err = db.Open(dir + "/db-" + types.String(i) + ".db")
-		if err != nil {
-			return err
-		}
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
 
-		err = db.Init()
-		if err != nil {
-			return err
-		}
+			var db = NewFileListDB()
+			dbErr := db.Open(dir + "/db-" + types.String(i) + ".db")
+			if dbErr != nil {
+				lastErr = dbErr
+				return
+			}
 
-		this.dbList[i] = db
+			dbErr = db.Init()
+			if dbErr != nil {
+				lastErr = dbErr
+				return
+			}
+
+			locker.Lock()
+			this.dbList[i] = db
+			locker.Unlock()
+		}(i)
+	}
+	wg.Wait()
+
+	if lastErr != nil {
+		return lastErr
 	}
 
 	// 升级老版本数据库
@@ -99,9 +122,7 @@ func (this *FileList) Add(hash string, item *Item) error {
 		return err
 	}
 
-	// 这里不增加点击量，以减少对数据库的操作次数
-
-	this.memoryCache.Write(hash, 1, item.ExpiredAt)
+	this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiredAt))
 
 	if this.onAdd != nil {
 		this.onAdd(item)
@@ -134,15 +155,27 @@ func (this *FileList) Exist(hash string) (bool, error) {
 	var expiredAt int64
 	err := row.Scan(&expiredAt)
 	if err != nil {
-		if err == sql.ErrNoRows {
+		if errors.Is(err, sql.ErrNoRows) {
 			err = nil
 		}
 		return false, err
 	}
-	this.memoryCache.Write(hash, 1, expiredAt)
+	this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(expiredAt))
 	return true, nil
 }
 
+func (this *FileList) ExistQuick(hash string) (isReady bool, found bool) {
+	var db = this.GetDB(hash)
+
+	if !db.IsReady() || !db.HashMapIsLoaded() {
+		return
+	}
+
+	isReady = true
+	found = db.hashMap.Exist(hash)
+	return
+}
+
 // CleanPrefix 清理某个前缀的缓存数据
 func (this *FileList) CleanPrefix(prefix string) error {
 	if len(prefix) == 0 {
@@ -204,7 +237,7 @@ func (this *FileList) CleanMatchPrefix(prefix string) error {
 }
 
 func (this *FileList) Remove(hash string) error {
-	_, err := this.remove(hash)
+	_, err := this.remove(hash, false)
 	return err
 }
 
@@ -223,11 +256,16 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
 		if err != nil {
 			return 0, nil
 		}
+
+		if len(hashStrings) == 0 {
+			continue
+		}
+
 		countFound += len(hashStrings)
 
 		// 不在 rows.Next() 循环中操作是为了避免死锁
 		for _, hash := range hashStrings {
-			err = this.Remove(hash)
+			_, err = this.remove(hash, true)
 			if err != nil {
 				return 0, err
 			}
@@ -237,6 +275,11 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
 				return 0, err
 			}
 		}
+
+		_, err = db.writeDB.Exec(`DELETE FROM "cacheItems" WHERE "hash" IN ('` + strings.Join(hashStrings, "', '") + `')`)
+		if err != nil {
+			return 0, err
+		}
 	}
 
 	return countFound, nil
@@ -254,24 +297,27 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
 			return err
 		}
 
+		if len(hashStrings) == 0 {
+			continue
+		}
+
 		// 不在 rows.Next() 循环中操作是为了避免死锁
 		for _, hash := range hashStrings {
-			notFound, err := this.remove(hash)
+			_, err = this.remove(hash, true)
 			if err != nil {
 				return err
 			}
-			if notFound {
-				err = db.DeleteHitAsync(hash)
-				if err != nil {
-					return db.WrapError(err)
-				}
-			}
 
 			err = callback(hash)
 			if err != nil {
 				return err
 			}
 		}
+
+		_, err = db.writeDB.Exec(`DELETE FROM "cacheItems" WHERE "hash" IN ('` + strings.Join(hashStrings, "', '") + `')`)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -372,7 +418,16 @@ func (this *FileList) GetDB(hash string) *FileListDB {
 	return this.dbList[fnv.HashString(hash)%CountFileDB]
 }
 
-func (this *FileList) remove(hash string) (notFound bool, err error) {
+func (this *FileList) HashMapIsLoaded() bool {
+	for _, db := range this.dbList {
+		if !db.HashMapIsLoaded() {
+			return false
+		}
+	}
+	return true
+}
+
+func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
 	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
@@ -388,14 +443,11 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
 	// 从缓存中删除
 	this.memoryCache.Delete(hash)
 
-	err = db.DeleteSync(hash)
-	if err != nil {
-		return false, db.WrapError(err)
-	}
-
-	err = db.DeleteHitAsync(hash)
-	if err != nil {
-		return false, db.WrapError(err)
+	if !isDeleted {
+		err = db.DeleteSync(hash)
+		if err != nil {
+			return false, db.WrapError(err)
+		}
 	}
 
 	if this.onRemove != nil {
@@ -428,7 +480,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 		remotelogs.Println("CACHE", "upgrading local database finished")
 	}()
 
-	db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
+	db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
 	if err != nil {
 		return err
 	}
@@ -493,9 +545,6 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 					MetaSize:   metaSize,
 					Host:       host,
 					ServerId:   serverId,
-					Week1Hits:  0,
-					Week2Hits:  0,
-					Week:       0,
 				})
 				if err != nil {
 					if brokenOnError {
@@ -515,3 +564,11 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 
 	return nil
 }
+
+func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
+	var maxTimestamp = fasttime.Now().Unix() + 3600
+	if expiresAt > maxTimestamp {
+		return maxTimestamp
+	}
+	return expiresAt
+}

internal/caches/list_file_db.go

@@ -6,18 +6,15 @@ import (
 	"errors"
 	"fmt"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/types"
-	timeutil "github.com/iwind/TeaGo/utils/time"
 	"net"
 	"net/url"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -29,15 +26,13 @@ type FileListDB struct {
 	readDB  *dbs.DB
 	writeDB *dbs.DB
 
-	writeBatch *dbs.Batch
-
 	hashMap *FileListHashMap
 
 	itemsTableName string
-	hitsTableName  string
 
-	isClosed bool
-	isReady  bool
+	isClosed        bool // 是否已关闭
+	isReady         bool // 是否已完成初始化
+	hashMapIsLoaded bool // Hash是否已加载
 
 	// cacheItems
 	existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
@@ -52,16 +47,10 @@ type FileListDB struct {
 	deleteByHashStmt *dbs.Stmt // 根据hash删除数据
 	deleteByHashSQL  string
 
-	statStmt            *dbs.Stmt // 统计
-	purgeStmt           *dbs.Stmt // 清理
-	deleteAllStmt       *dbs.Stmt // 删除所有数据
-	listOlderItemsStmt  *dbs.Stmt // 读取较早存储的缓存
-	updateAccessWeekSQL string    // 修改访问日期
-
-	// hits
-	insertHitSQL       string // 写入数据
-	increaseHitSQL     string // 增加点击量
-	deleteHitByHashSQL string // 根据hash删除数据
+	statStmt           *dbs.Stmt // 统计
+	purgeStmt          *dbs.Stmt // 清理
+	deleteAllStmt      *dbs.Stmt // 删除所有数据
+	listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
 }
 
 func NewFileListDB() *FileListDB {
@@ -74,15 +63,15 @@ func (this *FileListDB) Open(dbPath string) error {
 	this.dbPath = dbPath
 
 	// 动态调整Cache值
-	var cacheSize = 32000
+	var cacheSize = 512
 	var memoryGB = utils.SystemMemoryGB()
-	if memoryGB >= 8 {
-		cacheSize += 32000 * memoryGB / 8
+	if memoryGB >= 1 {
+		cacheSize = 256 * memoryGB
 	}
 
 	// write db
 	// 这里不能加 EXCLUSIVE 锁，不然异步事务可能会失败
-	writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
+	writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
 	if err != nil {
 		return fmt.Errorf("open write database failed: %w", err)
 	}
@@ -109,17 +98,7 @@ func (this *FileListDB) Open(dbPath string) error {
 		}
 	}
 
-	this.writeBatch = dbs.NewBatch(writeDB, 4)
-	this.writeBatch.OnFail(func(err error) {
-		remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error()+" ("+filepath.Base(this.dbPath)+")")
-	})
-
-	goman.New(func() {
-		this.writeBatch.Exec()
-	})
-
 	if teaconst.EnableDBStat {
-		this.writeBatch.EnableStat(true)
 		this.writeDB.EnableStat(true)
 	}
 
@@ -142,7 +121,6 @@ func (this *FileListDB) Open(dbPath string) error {
 
 func (this *FileListDB) Init() error {
 	this.itemsTableName = "cacheItems"
-	this.hitsTableName = "hits"
 
 	// 创建
 	var err = this.initTables(1)
@@ -156,7 +134,7 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt", "accessWeek") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
 	if err != nil {
 		return err
@@ -167,7 +145,7 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
+	this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>? ORDER BY id ASC LIMIT 2000`)
 	if err != nil {
 		return err
 	}
@@ -193,42 +171,15 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
+	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
 	if err != nil {
 		return err
 	}
 
-	this.updateAccessWeekSQL = `UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`
-
-	this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
-
-	this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
-
-	this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
-
 	this.isReady = true
 
 	// 加载HashMap
-	go func() {
-		err := this.hashMap.Load(this)
-		if err != nil {
-			remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
-
-			// 自动修复错误
-			// TODO 将来希望能尽可能恢复以往数据库中的内容
-			if strings.Contains(err.Error(), "database is closed") || strings.Contains(err.Error(), "database disk image is malformed") {
-				_ = this.Close()
-				this.deleteDB()
-				remotelogs.Println("LIST_FILE_DB", "recreating the database (file:"+this.dbPath+") ...")
-				err = this.Open(this.dbPath)
-				if err != nil {
-					remotelogs.Error("LIST_FILE_DB", "recreate the database failed: "+err.Error()+" (file:"+this.dbPath+")")
-				} else {
-					_ = this.Init()
-				}
-			}
-		}
-	}()
+	go this.loadHashMap()
 
 	return nil
 }
@@ -255,7 +206,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
 		item.StaleAt = item.ExpiredAt
 	}
 
-	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix(), timeutil.Format("YW"))
+	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
 	if err != nil {
 		return this.WrapError(err)
 	}
@@ -351,14 +302,7 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
 }
 
 func (this *FileListDB) IncreaseHitAsync(hash string) error {
-	var week = timeutil.Format("YW")
-	this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
-	this.writeBatch.Add(this.updateAccessWeekSQL, week, hash)
-	return nil
-}
-
-func (this *FileListDB) DeleteHitAsync(hash string) error {
-	this.writeBatch.Add(this.deleteHitByHashSQL, hash)
+	// do nothing
 	return nil
 }
 
@@ -367,10 +311,9 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
 		return nil
 	}
 	var count = int64(10000)
-	var staleLife = 600                  // TODO 需要可以设置
 	var unixTime = fasttime.Now().Unix() // 只删除当前的，不删除新的
 	for {
-		result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
+		result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+DefaultStaleCacheSeconds, unixTime, prefix)
 		if err != nil {
 			return this.WrapError(err)
 		}
@@ -414,15 +357,14 @@ func (this *FileListDB) CleanMatchKey(key string) error {
 	queryKey = strings.Replace(queryKey, "*", "%", 1)
 
 	// TODO 检查大批量数据下的操作性能
-	var staleLife = 600                  // TODO 需要可以设置
 	var unixTime = fasttime.Now().Unix() // 只删除当前的，不删除新的
 
-	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey)
+	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+DefaultStaleCacheSeconds, host, "*."+host, queryKey)
 	if err != nil {
 		return err
 	}
 
-	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey+SuffixAll+"%")
+	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+DefaultStaleCacheSeconds, host, "*."+host, queryKey+SuffixAll+"%")
 	if err != nil {
 		return err
 	}
@@ -456,10 +398,9 @@ func (this *FileListDB) CleanMatchPrefix(prefix string) error {
 	queryPrefix += "%"
 
 	// TODO 检查大批量数据下的操作性能
-	var staleLife = 600                  // TODO 需要可以设置
 	var unixTime = fasttime.Now().Unix() // 只删除当前的，不删除新的
 
-	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryPrefix)
+	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+DefaultStaleCacheSeconds, host, "*."+host, queryPrefix)
 	return err
 }
 
@@ -543,6 +484,10 @@ func (this *FileListDB) WrapError(err error) error {
 	return fmt.Errorf("%w (file: %s)", err, this.dbPath)
 }
 
+func (this *FileListDB) HashMapIsLoaded() bool {
+	return this.hashMapIsLoaded
+}
+
 // 初始化
 func (this *FileListDB) initTables(times int) error {
 	{
@@ -561,8 +506,7 @@ func (this *FileListDB) initTables(times int) error {
   "staleAt" integer DEFAULT 0,
   "createdAt" integer DEFAULT 0,
   "host" varchar(128),
-  "serverId" integer,
-  "accessWeek" varchar(6)
+  "serverId" integer
 );
 
 DROP INDEX IF EXISTS "createdAt";
@@ -578,8 +522,6 @@ CREATE INDEX IF NOT EXISTS "hash"
 ON "` + this.itemsTableName + `" (
   "hash" ASC
 );
-
-ALTER TABLE "cacheItems" ADD "accessWeek" varchar(6);
 `)
 
 		if err != nil {
@@ -603,32 +545,9 @@ ALTER TABLE "cacheItems" ADD "accessWeek" varchar(6);
 		}
 	}
 
+	// 删除hits表
 	{
-		_, err := this.writeDB.Exec(`CREATE TABLE IF NOT EXISTS "` + this.hitsTableName + `" (
-  "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
-  "hash" varchar(32),
-  "week1Hits" integer DEFAULT 0,
-  "week2Hits" integer DEFAULT 0,
-  "week" varchar(6)
-);
-
-CREATE UNIQUE INDEX IF NOT EXISTS "hits_hash"
-ON "` + this.hitsTableName + `" (
-  "hash" ASC
-);
-`)
-		if err != nil {
-			// 尝试删除重建
-			if times < 3 {
-				_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.hitsTableName + `"`)
-				if dropErr == nil {
-					return this.initTables(times + 1)
-				}
-				return this.WrapError(err)
-			}
-
-			return this.WrapError(err)
-		}
+		_, _ = this.writeDB.Exec(`DROP TABLE "hits"`)
 	}
 
 	return nil
@@ -678,3 +597,30 @@ func (this *FileListDB) deleteDB() {
 	_ = os.Remove(this.dbPath + "-shm")
 	_ = os.Remove(this.dbPath + "-wal")
 }
+
+// 加载Hash列表
+func (this *FileListDB) loadHashMap() {
+	this.hashMapIsLoaded = false
+
+	err := this.hashMap.Load(this)
+	if err != nil {
+		remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
+
+		// 自动修复错误
+		// TODO 将来希望能尽可能恢复以往数据库中的内容
+		if strings.Contains(err.Error(), "database is closed") || strings.Contains(err.Error(), "database disk image is malformed") {
+			_ = this.Close()
+			this.deleteDB()
+			remotelogs.Println("LIST_FILE_DB", "recreating the database (file:"+this.dbPath+") ...")
+			err = this.Open(this.dbPath)
+			if err != nil {
+				remotelogs.Error("LIST_FILE_DB", "recreate the database failed: "+err.Error()+" (file:"+this.dbPath+")")
+			} else {
+				_ = this.Init()
+			}
+		}
+		return
+	}
+
+	this.hashMapIsLoaded = true
+}

internal/caches/list_file_db_test.go

@@ -4,13 +4,20 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
+	"runtime"
+	"runtime/debug"
 	"testing"
 	"time"
 )
 
 func TestFileListDB_ListLFUItems(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = caches.NewFileListDB()
 
 	defer func() {
@@ -34,33 +41,11 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
 	t.Log("[", len(hashList), "]", hashList)
 }
 
-func TestFileListDB_IncreaseHitAsync(t *testing.T) {
-	var db = caches.NewFileListDB()
-
-	defer func() {
-		_ = db.Close()
-	}()
-
-	err := db.Open(Tea.Root + "/data/cache-db-large.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.Init()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = db.IncreaseHitAsync("4598e5231ba47d6ec7aa9ea640ff2eaf")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// wait transaction
-	time.Sleep(1 * time.Second)
-}
-
 func TestFileListDB_CleanMatchKey(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = caches.NewFileListDB()
 
 	defer func() {
@@ -89,6 +74,10 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
 }
 
 func TestFileListDB_CleanMatchPrefix(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = caches.NewFileListDB()
 
 	defer func() {
@@ -115,3 +104,67 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestFileListDB_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var db = caches.NewFileListDB()
+
+	defer func() {
+		_ = db.Close()
+	}()
+
+	err := db.Open(Tea.Root + "/data/cache-index/p1/db-0.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log(db.Total())
+
+	// load hashes
+	var maxId int64
+	var hashList []string
+	var before = time.Now()
+	for i := 0; i < 1_000; i++ {
+		hashList, maxId, err = db.ListHashes(maxId)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(hashList) == 0 {
+			t.Log("hashes loaded", time.Since(before).Seconds()*1000, "ms")
+			break
+		}
+		if i%100 == 0 {
+			t.Log(i)
+		}
+	}
+
+	runtime.GC()
+	debug.FreeOSMemory()
+
+	//time.Sleep(600 * time.Second)
+
+	for i := 0; i < 1_000; i++ {
+		_, err = db.ListLFUItems(5000)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i%100 == 0 {
+			t.Log(i)
+		}
+	}
+
+	t.Log("loaded")
+
+	runtime.GC()
+	debug.FreeOSMemory()
+
+	time.Sleep(600 * time.Second)
+}

internal/caches/list_file_hash_map.go

@@ -9,18 +9,36 @@ import (
 	"sync"
 )
 
+const HashMapSharding = 31
+
+var bigIntPool = sync.Pool{
+	New: func() any {
+		return big.NewInt(0)
+	},
+}
+
 // FileListHashMap 文件Hash列表
 type FileListHashMap struct {
-	m map[uint64]zero.Zero
+	m []map[uint64]zero.Zero
+
+	lockers []*sync.RWMutex
 
-	locker      sync.RWMutex
 	isAvailable bool
 	isReady     bool
 }
 
 func NewFileListHashMap() *FileListHashMap {
+	var m = make([]map[uint64]zero.Zero, HashMapSharding)
+	var lockers = make([]*sync.RWMutex, HashMapSharding)
+
+	for i := 0; i < HashMapSharding; i++ {
+		m[i] = map[uint64]zero.Zero{}
+		lockers[i] = &sync.RWMutex{}
+	}
+
 	return &FileListHashMap{
-		m:           map[uint64]zero.Zero{},
+		m:           m,
+		lockers:     lockers,
 		isAvailable: false,
 		isReady:     false,
 	}
@@ -35,6 +53,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
 	this.isAvailable = true
 
 	var lastId int64
+	var maxLoops = 50_000
 	for {
 		hashList, maxId, err := db.ListHashes(lastId)
 		if err != nil {
@@ -45,6 +64,11 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
 		}
 		this.AddHashes(hashList)
 		lastId = maxId
+
+		maxLoops--
+		if maxLoops <= 0 {
+			break
+		}
 	}
 
 	this.isReady = true
@@ -56,9 +80,11 @@ func (this *FileListHashMap) Add(hash string) {
 		return
 	}
 
-	this.locker.Lock()
-	this.m[this.bigInt(hash)] = zero.New()
-	this.locker.Unlock()
+	hashInt, index := this.bigInt(hash)
+
+	this.lockers[index].Lock()
+	this.m[index][hashInt] = zero.New()
+	this.lockers[index].Unlock()
 }
 
 func (this *FileListHashMap) AddHashes(hashes []string) {
@@ -66,11 +92,12 @@ func (this *FileListHashMap) AddHashes(hashes []string) {
 		return
 	}
 
-	this.locker.Lock()
 	for _, hash := range hashes {
-		this.m[this.bigInt(hash)] = zero.New()
+		hashInt, index := this.bigInt(hash)
+		this.lockers[index].Lock()
+		this.m[index][hashInt] = zero.New()
+		this.lockers[index].Unlock()
 	}
-	this.locker.Unlock()
 }
 
 func (this *FileListHashMap) Delete(hash string) {
@@ -78,9 +105,10 @@ func (this *FileListHashMap) Delete(hash string) {
 		return
 	}
 
-	this.locker.Lock()
-	delete(this.m, this.bigInt(hash))
-	this.locker.Unlock()
+	hashInt, index := this.bigInt(hash)
+	this.lockers[index].Lock()
+	delete(this.m[index], hashInt)
+	this.lockers[index].Unlock()
 }
 
 func (this *FileListHashMap) Exist(hash string) bool {
@@ -91,16 +119,25 @@ func (this *FileListHashMap) Exist(hash string) bool {
 		// 只有完全Ready时才能判断是否为false
 		return true
 	}
-	this.locker.RLock()
-	_, ok := this.m[this.bigInt(hash)]
-	this.locker.RUnlock()
+
+	hashInt, index := this.bigInt(hash)
+
+	this.lockers[index].RLock()
+	_, ok := this.m[index][hashInt]
+	this.lockers[index].RUnlock()
 	return ok
 }
 
 func (this *FileListHashMap) Clean() {
-	this.locker.Lock()
-	this.m = map[uint64]zero.Zero{}
-	this.locker.Unlock()
+	for i := 0; i < HashMapSharding; i++ {
+		this.lockers[i].Lock()
+	}
+
+	this.m = make([]map[uint64]zero.Zero, HashMapSharding)
+
+	for i := HashMapSharding - 1; i >= 0; i-- {
+		this.lockers[i].Unlock()
+	}
 }
 
 func (this *FileListHashMap) IsReady() bool {
@@ -108,13 +145,36 @@ func (this *FileListHashMap) IsReady() bool {
 }
 
 func (this *FileListHashMap) Len() int {
-	this.locker.Lock()
-	defer this.locker.Unlock()
-	return len(this.m)
+	for i := 0; i < HashMapSharding; i++ {
+		this.lockers[i].Lock()
+	}
+
+	var count = 0
+	for _, shard := range this.m {
+		count += len(shard)
+	}
+
+	for i := HashMapSharding - 1; i >= 0; i-- {
+		this.lockers[i].Unlock()
+	}
+
+	return count
 }
 
-func (this *FileListHashMap) bigInt(hash string) uint64 {
-	var bigInt = big.NewInt(0)
-	bigInt.SetString(hash, 16)
-	return bigInt.Uint64()
+func (this *FileListHashMap) SetIsAvailable(isAvailable bool) {
+	this.isAvailable = isAvailable
+}
+
+func (this *FileListHashMap) SetIsReady(isReady bool) {
+	this.isReady = isReady
+}
+
+func (this *FileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
+	var bigInt = bigIntPool.Get().(*big.Int)
+	bigInt.SetString(hash, 16)
+	hashInt = bigInt.Uint64()
+	bigIntPool.Put(bigInt)
+
+	index = int(hashInt % HashMapSharding)
+	return
 }

internal/caches/list_file_hash_map_test.go

@@ -6,6 +6,8 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/assert"
+	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
 	stringutil "github.com/iwind/TeaGo/utils/string"
 	"math/big"
@@ -20,15 +22,19 @@ func TestFileListHashMap_Memory(t *testing.T) {
 	runtime.ReadMemStats(stat1)
 
 	var m = caches.NewFileListHashMap()
+	m.SetIsAvailable(true)
 
 	for i := 0; i < 1_000_000; i++ {
 		m.Add(stringutil.Md5(types.String(i)))
 	}
 
+	t.Log("added:", m.Len(), "hashes")
+
 	var stat2 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat2)
 
 	t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
+	t.Log("remains:", m.Len(), "hashes")
 }
 
 func TestFileListHashMap_Memory2(t *testing.T) {
@@ -48,12 +54,34 @@ func TestFileListHashMap_Memory2(t *testing.T) {
 }
 
 func TestFileListHashMap_BigInt(t *testing.T) {
+	var bigInt = big.NewInt(0)
+
 	for _, s := range []string{"1", "2", "3", "123", "123456"} {
 		var hash = stringutil.Md5(s)
 
-		var bigInt = big.NewInt(0)
+		var bigInt1 = big.NewInt(0)
+		bigInt1.SetString(hash, 16)
+
 		bigInt.SetString(hash, 16)
-		t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
+
+		t.Log(s, "=>", bigInt1.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt1.Uint64(), 16), strconv.FormatUint(bigInt.Uint64(), 16))
+
+		if strconv.FormatUint(bigInt1.Uint64(), 16) != strconv.FormatUint(bigInt.Uint64(), 16) {
+			t.Fatal("not equal")
+		}
+	}
+
+	for i := 0; i < 1_000_000; i++ {
+		var hash = stringutil.Md5(types.String(i))
+
+		var bigInt1 = big.NewInt(0)
+		bigInt1.SetString(hash, 16)
+
+		bigInt.SetString(hash, 16)
+
+		if bigInt1.Uint64() != bigInt.Uint64() {
+			t.Fatal(i, "not equal")
+		}
 	}
 }
 
@@ -85,6 +113,18 @@ func TestFileListHashMap_Load(t *testing.T) {
 	}
 }
 
+func TestFileListHashMap_Delete(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var m = caches.NewFileListHashMap()
+	m.SetIsReady(true)
+	m.SetIsAvailable(true)
+	m.Add("a")
+	a.IsTrue(m.Len() == 1)
+	m.Delete("a")
+	a.IsTrue(m.Len() == 0)
+}
+
 func Benchmark_BigInt(b *testing.B) {
 	var hash = stringutil.Md5("123456")
 	b.ResetTimer()
@@ -95,3 +135,23 @@ func Benchmark_BigInt(b *testing.B) {
 		_ = bigInt.Uint64()
 	}
 }
+
+func BenchmarkFileListHashMap_Exist(b *testing.B) {
+	var m = caches.NewFileListHashMap()
+	m.SetIsAvailable(true)
+	m.SetIsReady(true)
+
+	for i := 0; i < 1_000_000; i++ {
+		m.Add(types.String(i))
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			m.Add(types.String(rands.Int64()))
+			_ = m.Exist(types.String(rands.Int64()))
+		}
+	})
+}

internal/caches/list_file_test.go

@@ -281,11 +281,6 @@ func TestFileList_PurgeLFU(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = list.IncreaseHit(stringutil.Md5("123456"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	var count = 0
 	err = list.PurgeLFU(caches.CountFileDB*2, func(hash string) error {
 		t.Log(hash)
@@ -356,41 +351,6 @@ func TestFileList_CleanAll(t *testing.T) {
 	t.Log(list.Count())
 }
 
-func TestFileList_IncreaseHit(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
-
-	defer func() {
-		_ = list.Close()
-	}()
-
-	err := list.Init()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func() {
-		_ = list.Close()
-	}()
-
-	var before = time.Now()
-	defer func() {
-		t.Log(time.Since(before).Seconds()*1000, "ms")
-	}()
-
-	var count = 1_000_000
-
-	if !testutils.IsSingleTesting() {
-		count = 10
-	}
-	for i := 0; i < count; i++ {
-		err = list.IncreaseHit(stringutil.Md5("abc" + types.String(i)))
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok")
-}
-
 func TestFileList_UpgradeV3(t *testing.T) {
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList)
 

internal/caches/list_memory.go

@@ -2,7 +2,6 @@ package caches
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
-	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/logs"
 	"net"
 	"net/url"
@@ -19,9 +18,6 @@ type MemoryList struct {
 
 	itemMaps map[string]map[string]*Item // prefix => { hash => item }
 
-	weekItemMaps map[int32]map[string]zero.Zero // week => { hash => Zero }
-	minWeek      int32
-
 	prefixes []string
 	locker   sync.RWMutex
 	onAdd    func(item *Item)
@@ -32,9 +28,7 @@ type MemoryList struct {
 
 func NewMemoryList() ListInterface {
 	return &MemoryList{
-		itemMaps:     map[string]map[string]*Item{},
-		weekItemMaps: map[int32]map[string]zero.Zero{},
-		minWeek:      currentWeek(),
+		itemMaps: map[string]map[string]*Item{},
 	}
 }
 
@@ -56,7 +50,6 @@ func (this *MemoryList) Reset() error {
 	for key := range this.itemMaps {
 		this.itemMaps[key] = map[string]*Item{}
 	}
-	this.weekItemMaps = map[int32]map[string]zero.Zero{}
 	this.locker.Unlock()
 
 	atomic.StoreInt64(&this.count, 0)
@@ -65,10 +58,6 @@ func (this *MemoryList) Reset() error {
 }
 
 func (this *MemoryList) Add(hash string, item *Item) error {
-	if item.Week == 0 {
-		item.Week = currentWeek()
-	}
-
 	this.locker.Lock()
 
 	prefix := this.prefix(hash)
@@ -81,14 +70,6 @@ func (this *MemoryList) Add(hash string, item *Item) error {
 	// 先删除，为了可以正确触发统计
 	oldItem, ok := itemMap[hash]
 	if ok {
-		// 从week map中删除
-		if oldItem.Week > 0 {
-			wm, ok := this.weekItemMaps[oldItem.Week]
-			if ok {
-				delete(wm, hash)
-			}
-		}
-
 		// 回调
 		if this.onRemove != nil {
 			this.onRemove(oldItem)
@@ -104,14 +85,6 @@ func (this *MemoryList) Add(hash string, item *Item) error {
 
 	itemMap[hash] = item
 
-	// week map
-	wm, ok := this.weekItemMaps[item.Week]
-	if ok {
-		wm[hash] = zero.New()
-	} else {
-		this.weekItemMaps[item.Week] = map[string]zero.Zero{hash: zero.New()}
-	}
-
 	this.locker.Unlock()
 	return nil
 }
@@ -242,14 +215,6 @@ func (this *MemoryList) Remove(hash string) error {
 
 		atomic.AddInt64(&this.count, -1)
 		delete(itemMap, hash)
-
-		// week map
-		if item.Week > 0 {
-			wm, ok := this.weekItemMaps[item.Week]
-			if ok {
-				delete(wm, hash)
-			}
-		}
 	}
 
 	this.locker.Unlock()
@@ -261,12 +226,12 @@ func (this *MemoryList) Remove(hash string) error {
 // callback 每次发现过期key的调用
 func (this *MemoryList) Purge(count int, callback func(hash string) error) (int, error) {
 	this.locker.Lock()
-	deletedHashList := []string{}
+	var deletedHashList = []string{}
 
 	if this.purgeIndex >= len(this.prefixes) {
 		this.purgeIndex = 0
 	}
-	prefix := this.prefixes[this.purgeIndex]
+	var prefix = this.prefixes[this.purgeIndex]
 
 	this.purgeIndex++
 
@@ -290,14 +255,6 @@ func (this *MemoryList) Purge(count int, callback func(hash string) error) (int,
 			delete(itemMap, hash)
 			deletedHashList = append(deletedHashList, hash)
 
-			// week map
-			if item.Week > 0 {
-				wm, ok := this.weekItemMaps[item.Week]
-				if ok {
-					delete(wm, hash)
-				}
-			}
-
 			countFound++
 		}
 
@@ -322,63 +279,48 @@ func (this *MemoryList) PurgeLFU(count int, callback func(hash string) error) er
 		return nil
 	}
 
-	var week = currentWeek()
-	if this.minWeek > week {
-		this.minWeek = week
-	}
-
 	var deletedHashList = []string{}
 
+	var week = currentWeek()
+	var round = 0
+
+	this.locker.Lock()
+
 Loop:
-	for w := this.minWeek; w <= week; w++ {
-		this.minWeek = w
+	for {
+		var found = false
+		round++
+		for _, itemMap := range this.itemMaps {
+			for hash, item := range itemMap {
+				found = true
 
-		this.locker.Lock()
-		wm, ok := this.weekItemMaps[w]
-		if ok {
-			var wc = len(wm)
-			if wc == 0 {
-				delete(this.weekItemMaps, w)
-			} else {
-				if wc <= count {
-					delete(this.weekItemMaps, w)
+				if week-item.Week <= 1 /** 最近有在使用 **/ && round <= 3 /** 查找轮数过多还不满足数量要求的就不再限制 **/ {
+					continue
 				}
 
-				// TODO 未来支持按照点击量排序
-				for hash := range wm {
-					count--
-
-					if count < 0 {
-						this.locker.Unlock()
-						break Loop
-					}
-
-					delete(wm, hash)
-
-					itemMap, ok := this.itemMaps[this.prefix(hash)]
-					if !ok {
-						continue
-					}
-					item, ok := itemMap[hash]
-					if !ok {
-						continue
-					}
-
-					if this.onRemove != nil {
-						this.onRemove(item)
-					}
-
-					atomic.AddInt64(&this.count, -1)
-					delete(itemMap, hash)
-					deletedHashList = append(deletedHashList, hash)
+				if this.onRemove != nil {
+					this.onRemove(item)
 				}
+
+				atomic.AddInt64(&this.count, -1)
+				delete(itemMap, hash)
+				deletedHashList = append(deletedHashList, hash)
+
+				count--
+				if count <= 0 {
+					break Loop
+				}
+
+				break
 			}
-		} else {
-			delete(this.weekItemMaps, w)
 		}
-		this.locker.Unlock()
+		if !found {
+			break
+		}
 	}
 
+	this.locker.Unlock()
+
 	// 执行外部操作
 	for _, hash := range deletedHashList {
 		if callback != nil {
@@ -451,23 +393,7 @@ func (this *MemoryList) IncreaseHit(hash string) error {
 
 	item, ok := itemMap[hash]
 	if ok {
-		var week = currentWeek()
-
-		// 交换位置
-		if item.Week > 0 && item.Week != week {
-			wm, ok := this.weekItemMaps[item.Week]
-			if ok {
-				delete(wm, hash)
-			}
-			wm, ok = this.weekItemMaps[week]
-			if ok {
-				wm[hash] = zero.New()
-			} else {
-				this.weekItemMaps[week] = map[string]zero.Zero{hash: zero.New()}
-			}
-		}
-
-		item.IncreaseHit(week)
+		item.Week = currentWeek()
 	}
 
 	this.locker.Unlock()

internal/caches/list_memory_test.go

@@ -6,7 +6,9 @@ import (
 	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	"math/rand"
+	"sort"
 	"strconv"
 	"testing"
 	"time"
@@ -32,7 +34,6 @@ func TestMemoryList_Add(t *testing.T) {
 	})
 	t.Log(list.prefixes)
 	logs.PrintAsJSON(list.itemMaps, t)
-	logs.PrintAsJSON(list.weekItemMaps, t)
 	t.Log(list.Count())
 }
 
@@ -51,7 +52,6 @@ func TestMemoryList_Remove(t *testing.T) {
 	})
 	_ = list.Remove("b")
 	list.print(t)
-	logs.PrintAsJSON(list.weekItemMaps, t)
 	t.Log(list.Count())
 }
 
@@ -83,7 +83,6 @@ func TestMemoryList_Purge(t *testing.T) {
 		return nil
 	})
 	list.print(t)
-	logs.PrintAsJSON(list.weekItemMaps, t)
 
 	for i := 0; i < 1000; i++ {
 		_, _ = list.Purge(100, func(hash string) error {
@@ -172,27 +171,64 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
 	t.Log(time.Since(before).Seconds()*1000, "ms")
 }
 
+func TestMapRandomDelete(t *testing.T) {
+	var countMap = map[int]int{} // k => count
+
+	for j := 0; j < 1_000_000; j++ {
+		var m = map[int]bool{}
+		for i := 0; i < 100; i++ {
+			m[i] = true
+		}
+
+		var count = 0
+		for k := range m {
+			delete(m, k)
+			count++
+			if count >= 10 {
+				break
+			}
+		}
+
+		for k := range m {
+			countMap[k]++
+		}
+	}
+
+	var counts = []int{}
+	for _, count := range countMap {
+		counts = append(counts, count)
+	}
+	sort.Ints(counts)
+	t.Log("["+types.String(len(counts))+"]", counts)
+}
+
 func TestMemoryList_PurgeLFU(t *testing.T) {
 	var list = NewMemoryList().(*MemoryList)
-	list.minWeek = 2704
 
 	var before = time.Now()
 	defer func() {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}()
 
-	t.Log("current week:", currentWeek())
-
 	_ = list.Add("1", &Item{})
 	_ = list.Add("2", &Item{})
 	_ = list.Add("3", &Item{})
 	_ = list.Add("4", &Item{})
 	_ = list.Add("5", &Item{})
-	_ = list.Add("6", &Item{Week: 2704})
-	_ = list.Add("7", &Item{Week: 2704})
-	_ = list.Add("8", &Item{Week: 2705})
 
-	err := list.PurgeLFU(2, func(hash string) error {
+	//_ = list.IncreaseHit("1")
+	//_ = list.IncreaseHit("2")
+	//_ = list.IncreaseHit("3")
+	//_ = list.IncreaseHit("4")
+	//_ = list.IncreaseHit("5")
+
+	count, err := list.Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("count items before purge:", count)
+
+	err = list.PurgeLFU(5, func(hash string) error {
 		t.Log("purge lfu:", hash)
 		return nil
 	})
@@ -201,40 +237,18 @@ func TestMemoryList_PurgeLFU(t *testing.T) {
 	}
 	t.Log("ok")
 
-	logs.PrintAsJSON(list.weekItemMaps, t)
-	t.Log(list.Count())
-}
-
-func TestMemoryList_IncreaseHit(t *testing.T) {
-	var list = NewMemoryList().(*MemoryList)
-	var item = &Item{}
-	item.Week = 2705
-	item.Week2Hits = 100
-
-	_ = list.Add("a", &Item{})
-	_ = list.Add("a", item)
-	t.Log("hits1:", item.Week1Hits, "hits2:", item.Week2Hits, "week:", item.Week)
-	logs.PrintAsJSON(list.weekItemMaps, t)
-
-	_ = list.IncreaseHit("a")
-	t.Log("hits1:", item.Week1Hits, "hits2:", item.Week2Hits, "week:", item.Week)
-	logs.PrintAsJSON(list.weekItemMaps, t)
-
-	_ = list.IncreaseHit("a")
-	t.Log("hits1:", item.Week1Hits, "hits2:", item.Week2Hits, "week:", item.Week)
-	logs.PrintAsJSON(list.weekItemMaps, t)
+	count, err = list.Count()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("count items left:", count)
 }
 
 func TestMemoryList_CleanAll(t *testing.T) {
 	var list = NewMemoryList().(*MemoryList)
-	var item = &Item{}
-	item.Week = 2705
-	item.Week2Hits = 100
-
 	_ = list.Add("a", &Item{})
 	_ = list.CleanAll()
 	logs.PrintAsJSON(list.itemMaps, t)
-	logs.PrintAsJSON(list.weekItemMaps, t)
 	t.Log(list.Count())
 }
 

internal/caches/manager.go

@@ -7,6 +7,7 @@ import (
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"golang.org/x/sys/unix"
@@ -35,6 +36,9 @@ type Manager struct {
 	SubDiskDirs       []*serverconfigs.CacheDir
 	MaxMemoryCapacity *shared.SizeCapacity
 
+	CountFileStorages   int
+	CountMemoryStorages int
+
 	policyMap  map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
 	storageMap map[int64]StorageInterface               // policyId => *Storage
 	locker     sync.RWMutex
@@ -143,6 +147,16 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
 			}
 		}
 	}
+
+	this.CountFileStorages = 0
+	this.CountFileStorages = 0
+	for _, storage := range this.storageMap {
+		_, isFileStorage := storage.(*FileStorage)
+		this.CountMemoryStorages++
+		if isFileStorage {
+			this.CountFileStorages++
+		}
+	}
 }
 
 // FindPolicy 获取Policy信息
@@ -172,6 +186,11 @@ func (this *Manager) NewStorageWithPolicy(policy *serverconfigs.HTTPCachePolicy)
 	return nil
 }
 
+// StorageMap 获取已有的存储对象
+func (this *Manager) StorageMap() map[int64]StorageInterface {
+	return this.storageMap
+}
+
 // TotalDiskSize 消耗的磁盘尺寸
 func (this *Manager) TotalDiskSize() int64 {
 	this.locker.RLock()
@@ -256,3 +275,33 @@ func (this *Manager) FindAllStorages() []StorageInterface {
 	}
 	return storages
 }
+
+// ScanGarbageCaches 清理目录中“失联”的缓存文件
+func (this *Manager) ScanGarbageCaches(callback func(path string) error) error {
+	var storages = this.FindAllStorages()
+	for _, storage := range storages {
+		fileStorage, ok := storage.(*FileStorage)
+		if !ok {
+			continue
+		}
+		err := fileStorage.ScanGarbageCaches(callback)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// MaxSystemMemoryBytesPerStorage 计算单个策略能使用的系统最大内存
+func (this *Manager) MaxSystemMemoryBytesPerStorage() int64 {
+	var count = this.CountMemoryStorages
+	if count < 1 {
+		count = 1
+	}
+
+	var resultBytes = int64(utils.SystemMemoryBytes()) / 3 / int64(count) // 1/3 of the system memory
+	if resultBytes < 1<<30 {
+		resultBytes = 1 << 30
+	}
+	return resultBytes
+}

internal/caches/manager_test.go

@@ -1,8 +1,9 @@
-package caches
+package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/iwind/TeaGo/Tea"
 	"testing"
 )
@@ -10,7 +11,7 @@ import (
 func TestManager_UpdatePolicies(t *testing.T) {
 	{
 		var policies = []*serverconfigs.HTTPCachePolicy{}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 
@@ -38,7 +39,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
 				},
 			},
 		}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 
@@ -66,7 +67,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
 				},
 			},
 		}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 }
@@ -80,8 +81,8 @@ func TestManager_ChangePolicy_Memory(t *testing.T) {
 			Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
 		},
 	}
-	SharedManager.UpdatePolicies(policies)
-	SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
+	caches.SharedManager.UpdatePolicies(policies)
+	caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
 		{
 			Id:       1,
 			Type:     serverconfigs.CachePolicyStorageMemory,
@@ -102,8 +103,8 @@ func TestManager_ChangePolicy_File(t *testing.T) {
 			Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
 		},
 	}
-	SharedManager.UpdatePolicies(policies)
-	SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
+	caches.SharedManager.UpdatePolicies(policies)
+	caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
 		{
 			Id:   1,
 			Type: serverconfigs.CachePolicyStorageFile,
@@ -115,10 +116,17 @@ func TestManager_ChangePolicy_File(t *testing.T) {
 	})
 }
 
+func TestManager_MaxSystemMemoryBytesPerStorage(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		caches.SharedManager.CountMemoryStorages = i
+		t.Log(i, caches.SharedManager.MaxSystemMemoryBytesPerStorage()>>30, "GB")
+	}
+}
+
 func printManager(t *testing.T) {
 	t.Log("===manager==")
 	t.Log("storage:")
-	for _, storage := range SharedManager.storageMap {
+	for _, storage := range caches.SharedManager.StorageMap() {
 		t.Log("  storage:", storage.Policy().Id)
 	}
 	t.Log("===============")

internal/caches/memory_fragment_pool.go

@@ -0,0 +1,374 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import (
+	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
+	"github.com/TeaOSLab/EdgeNode/internal/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/iwind/TeaGo/logs"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	minMemoryFragmentPoolItemSize       = 8 << 10
+	maxMemoryFragmentPoolItemSize       = 128 << 20
+	maxItemsInMemoryFragmentPoolBucket  = 1024
+	memoryFragmentPoolBucketSegmentSize = 512 << 10
+	maxMemoryFragmentPoolItemAgeSeconds = 60
+)
+
+var SharedFragmentMemoryPool *MemoryFragmentPool
+
+func init() {
+	if !teaconst.IsMain {
+		return
+	}
+
+	SharedFragmentMemoryPool = NewMemoryFragmentPool()
+
+	goman.New(func() {
+		var ticker = time.NewTicker(200 * time.Millisecond)
+		for range ticker.C {
+			for i := 0; i < 10; i++ { // skip N empty buckets
+				var isEmpty = SharedFragmentMemoryPool.GCNextBucket()
+				if !isEmpty {
+					break
+				}
+			}
+		}
+	})
+}
+
+type MemoryFragmentPoolItem struct {
+	Bytes []byte
+
+	size      int64
+	createdAt int64
+
+	Refs int32
+}
+
+func (this *MemoryFragmentPoolItem) IsExpired() bool {
+	return this.createdAt < fasttime.Now().Unix()-maxMemoryFragmentPoolItemAgeSeconds
+}
+
+func (this *MemoryFragmentPoolItem) Reset() {
+	this.Bytes = nil
+}
+
+func (this *MemoryFragmentPoolItem) IsAvailable() bool {
+	return atomic.AddInt32(&this.Refs, 1) == 1
+}
+
+// MemoryFragmentPool memory fragments management
+type MemoryFragmentPool struct {
+	bucketMaps    []map[uint64]*MemoryFragmentPoolItem // [  { id => Zero }, ... ]
+	countBuckets  int
+	gcBucketIndex int
+
+	mu sync.RWMutex
+
+	id          uint64
+	totalMemory int64
+
+	isOk     bool
+	capacity int64
+
+	debugMode bool
+	countGet  uint64
+	countNew  uint64
+}
+
+// NewMemoryFragmentPool create new fragment memory pool
+func NewMemoryFragmentPool() *MemoryFragmentPool {
+	var pool = &MemoryFragmentPool{}
+	pool.init()
+	return pool
+}
+
+func (this *MemoryFragmentPool) init() {
+	var capacity = int64(utils.SystemMemoryGB()) << 30 / 16
+	if capacity > 256<<20 {
+		this.isOk = true
+		this.capacity = capacity
+
+		this.bucketMaps = []map[uint64]*MemoryFragmentPoolItem{}
+		for i := 0; i < maxMemoryFragmentPoolItemSize/memoryFragmentPoolBucketSegmentSize+1; i++ {
+			this.bucketMaps = append(this.bucketMaps, map[uint64]*MemoryFragmentPoolItem{})
+		}
+		this.countBuckets = len(this.bucketMaps)
+	}
+
+	// print statistics for debug
+	if len(os.Getenv("GOEDGE_DEBUG_MEMORY_FRAGMENT_POOL")) > 0 {
+		this.debugMode = true
+
+		go func() {
+			var maxRounds = 10_000
+			var ticker = time.NewTicker(10 * time.Second)
+			for range ticker.C {
+				logs.Println("reused:", this.countGet, "created:", this.countNew, "fragments:", this.Len(), "memory:", this.totalMemory>>20, "MB")
+
+				maxRounds--
+				if maxRounds <= 0 {
+					break
+				}
+			}
+		}()
+	}
+}
+
+// Get try to get a bytes object
+func (this *MemoryFragmentPool) Get(expectSize int64) (resultBytes []byte, ok bool) {
+	if !this.isOk {
+		return
+	}
+
+	if expectSize <= 0 {
+		return
+	}
+
+	// DO NOT check min segment size
+
+	this.mu.RLock()
+
+	var bucketIndex = this.bucketIndexForSize(expectSize)
+	var resultItemId uint64
+	const maxSearchingBuckets = 20
+	for i := bucketIndex; i <= bucketIndex+maxSearchingBuckets; i++ {
+		resultBytes, resultItemId, ok = this.findItemInMap(this.bucketMaps[i], expectSize)
+		if ok {
+			this.mu.RUnlock()
+
+			// remove from bucket
+			this.mu.Lock()
+			delete(this.bucketMaps[i], resultItemId)
+			this.mu.Unlock()
+
+			return
+		}
+		if i >= this.countBuckets {
+			break
+		}
+	}
+	this.mu.RUnlock()
+
+	return
+}
+
+// Put a bytes object to specified bucket
+func (this *MemoryFragmentPool) Put(data []byte) (ok bool) {
+	if !this.isOk {
+		return
+	}
+
+	var l = int64(cap(data)) // MUST be 'cap' instead of 'len'
+
+	if l < minMemoryFragmentPoolItemSize || l > maxMemoryFragmentPoolItemSize {
+		return
+	}
+
+	if atomic.LoadInt64(&this.totalMemory) >= this.capacity {
+		return
+	}
+
+	var itemId = atomic.AddUint64(&this.id, 1)
+
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	var bucketMap = this.bucketMaps[this.bucketIndexForSize(l)]
+	if len(bucketMap) >= maxItemsInMemoryFragmentPoolBucket {
+		return
+	}
+
+	atomic.AddInt64(&this.totalMemory, l)
+
+	bucketMap[itemId] = &MemoryFragmentPoolItem{
+		Bytes:     data,
+		size:      l,
+		createdAt: fasttime.Now().Unix(),
+	}
+
+	return true
+}
+
+// GC fully GC
+func (this *MemoryFragmentPool) GC() {
+	if !this.isOk {
+		return
+	}
+
+	var totalMemory = atomic.LoadInt64(&this.totalMemory)
+	if totalMemory < this.capacity {
+		return
+	}
+
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	var garbageSize = totalMemory * 1 / 10 // 10%
+
+	// remove expired
+	for _, bucketMap := range this.bucketMaps {
+		for itemId, item := range bucketMap {
+			if item.IsExpired() {
+				delete(bucketMap, itemId)
+				item.Reset()
+				atomic.AddInt64(&this.totalMemory, -item.size)
+
+				garbageSize -= item.size
+			}
+		}
+	}
+
+	// remove others
+	if garbageSize > 0 {
+		for _, bucketMap := range this.bucketMaps {
+			for itemId, item := range bucketMap {
+				delete(bucketMap, itemId)
+				item.Reset()
+				atomic.AddInt64(&this.totalMemory, -item.size)
+
+				garbageSize -= item.size
+				if garbageSize <= 0 {
+					break
+				}
+			}
+		}
+	}
+}
+
+// GCNextBucket gc one bucket
+func (this *MemoryFragmentPool) GCNextBucket() (isEmpty bool) {
+	if !this.isOk {
+		return
+	}
+
+	var itemIds = []uint64{}
+
+	// find
+	this.mu.RLock()
+
+	var bucketIndex = this.gcBucketIndex
+	var bucketMap = this.bucketMaps[bucketIndex]
+	isEmpty = len(bucketMap) == 0
+	if isEmpty {
+		this.mu.RUnlock()
+
+		// move to next bucket index
+		bucketIndex++
+		if bucketIndex >= this.countBuckets {
+			bucketIndex = 0
+		}
+		this.gcBucketIndex = bucketIndex
+
+		return
+	}
+
+	for itemId, item := range bucketMap {
+		if item.IsExpired() {
+			itemIds = append(itemIds, itemId)
+		}
+	}
+
+	this.mu.RUnlock()
+
+	// remove
+	if len(itemIds) > 0 {
+		this.mu.Lock()
+		for _, itemId := range itemIds {
+			item, ok := bucketMap[itemId]
+			if !ok {
+				continue
+			}
+			if !item.IsAvailable() {
+				continue
+			}
+			delete(bucketMap, itemId)
+			item.Reset()
+			atomic.AddInt64(&this.totalMemory, -item.size)
+		}
+		this.mu.Unlock()
+	}
+
+	// move to next bucket index
+	bucketIndex++
+	if bucketIndex >= this.countBuckets {
+		bucketIndex = 0
+	}
+	this.gcBucketIndex = bucketIndex
+
+	return
+}
+
+func (this *MemoryFragmentPool) SetCapacity(capacity int64) {
+	this.capacity = capacity
+}
+
+func (this *MemoryFragmentPool) TotalSize() int64 {
+	return atomic.LoadInt64(&this.totalMemory)
+}
+
+func (this *MemoryFragmentPool) Len() int {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+	var count = 0
+	for _, bucketMap := range this.bucketMaps {
+		count += len(bucketMap)
+	}
+	return count
+}
+
+func (this *MemoryFragmentPool) IncreaseNew() {
+	if this.isOk && this.debugMode {
+		atomic.AddUint64(&this.countNew, 1)
+	}
+}
+
+func (this *MemoryFragmentPool) bucketIndexForSize(size int64) int {
+	return int(size / memoryFragmentPoolBucketSegmentSize)
+}
+
+func (this *MemoryFragmentPool) findItemInMap(bucketMap map[uint64]*MemoryFragmentPoolItem, expectSize int64) (resultBytes []byte, resultItemId uint64, ok bool) {
+	if len(bucketMap) == 0 {
+		return
+	}
+
+	for itemId, item := range bucketMap {
+		if item.size >= expectSize {
+			// check if is referred
+			if !item.IsAvailable() {
+				continue
+			}
+
+			// return result
+			if item.size != expectSize {
+				resultBytes = item.Bytes[:expectSize]
+			} else {
+				resultBytes = item.Bytes
+			}
+
+			// reset old item
+			item.Reset()
+			atomic.AddInt64(&this.totalMemory, -item.size)
+
+			resultItemId = itemId
+
+			if this.debugMode {
+				atomic.AddUint64(&this.countGet, 1)
+			}
+
+			ok = true
+
+			return
+		}
+	}
+
+	return
+}

internal/caches/memory_fragment_pool_test.go

@@ -0,0 +1,313 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"bytes"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/assert"
+	"github.com/iwind/TeaGo/rands"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestNewMemoryFragmentPool(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var pool = caches.NewMemoryFragmentPool()
+	for i := 0; i < 3000; i++ {
+		ok := pool.Put(make([]byte, 2<<20))
+		if !ok {
+			t.Log("finished at", i)
+			break
+		}
+	}
+
+	t.Log(pool.TotalSize()>>20, "MB", pool.Len(), "items")
+
+	{
+		r, ok := pool.Get(1 << 20)
+		a.IsTrue(ok)
+		a.IsTrue(len(r) == 1<<20)
+	}
+
+	{
+		r, ok := pool.Get(2 << 20)
+		a.IsTrue(ok)
+		a.IsTrue(len(r) == 2<<20)
+	}
+
+	{
+		r, ok := pool.Get(4 << 20)
+		a.IsFalse(ok)
+		a.IsTrue(len(r) == 0)
+	}
+
+	t.Log(pool.TotalSize()>>20, "MB", pool.Len(), "items")
+}
+
+func TestNewMemoryFragmentPool_LargeBucket(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var pool = caches.NewMemoryFragmentPool()
+	{
+		pool.Put(make([]byte, 128<<20+1))
+		a.IsTrue(pool.Len() == 0)
+	}
+
+	{
+		pool.Put(make([]byte, 128<<20))
+		a.IsTrue(pool.Len() == 1)
+
+		pool.Get(118 << 20)
+		a.IsTrue(pool.Len() == 0)
+	}
+
+	{
+		pool.Put(make([]byte, 128<<20))
+		a.IsTrue(pool.Len() == 1)
+
+		pool.Get(110 << 20)
+		a.IsTrue(pool.Len() == 1)
+	}
+}
+
+func TestMemoryFragmentPool_Get_Exactly(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var pool = caches.NewMemoryFragmentPool()
+	{
+		pool.Put(make([]byte, 129<<20))
+		a.IsTrue(pool.Len() == 0)
+	}
+
+	{
+		pool.Put(make([]byte, 4<<20))
+		a.IsTrue(pool.Len() == 1)
+	}
+
+	{
+		pool.Get(4 << 20)
+		a.IsTrue(pool.Len() == 0)
+	}
+}
+
+func TestMemoryFragmentPool_Get_Round(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var pool = caches.NewMemoryFragmentPool()
+	{
+		pool.Put(make([]byte, 8<<20))
+		pool.Put(make([]byte, 8<<20))
+		pool.Put(make([]byte, 8<<20))
+		a.IsTrue(pool.Len() == 3)
+	}
+
+	{
+		resultBytes, ok := pool.Get(3 << 20)
+		a.IsTrue(pool.Len() == 2)
+		if ok {
+			pool.Put(resultBytes)
+		}
+	}
+
+	{
+		pool.Get(2 << 20)
+		a.IsTrue(pool.Len() == 2)
+	}
+
+	{
+		pool.Get(1 << 20)
+		a.IsTrue(pool.Len() == 1)
+	}
+}
+
+func TestMemoryFragmentPool_GC(t *testing.T) {
+	var pool = caches.NewMemoryFragmentPool()
+	pool.SetCapacity(32 << 20)
+	for i := 0; i < 16; i++ {
+		pool.Put(make([]byte, 4<<20))
+	}
+	var before = time.Now()
+	pool.GC()
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+	t.Log(pool.Len())
+}
+
+func TestMemoryFragmentPool_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var pool = caches.NewMemoryFragmentPool()
+
+	testutils.StartMemoryStats(t, func() {
+		t.Log(pool.Len(), "items")
+	})
+
+	var sampleData = bytes.Repeat([]byte{'A'}, 16<<20)
+
+	var countNew = 0
+	for i := 0; i < 1000; i++ {
+		cacheData, ok := pool.Get(16 << 20)
+		if ok {
+			copy(cacheData, sampleData)
+			pool.Put(cacheData)
+		} else {
+			countNew++
+			var data = make([]byte, 16<<20)
+			copy(data, sampleData)
+			pool.Put(data)
+		}
+	}
+
+	t.Log("count new:", countNew)
+	t.Log("count remains:", pool.Len())
+
+	time.Sleep(10 * time.Minute)
+}
+
+func TestMemoryFragmentPool_GCNextBucket(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var pool = caches.NewMemoryFragmentPool()
+	for i := 0; i < 1000; i++ {
+		pool.Put(make([]byte, rands.Int(0, 100)<<20))
+	}
+
+	var lastLen int
+	for {
+		pool.GCNextBucket()
+		var currentLen = pool.Len()
+		if lastLen == currentLen {
+			continue
+		}
+		lastLen = currentLen
+
+		t.Log(currentLen, "items", pool.TotalSize(), "bytes", timeutil.Format("H:i:s"))
+		time.Sleep(100 * time.Millisecond)
+
+		if currentLen == 0 {
+			break
+		}
+	}
+}
+
+func TestMemoryFragmentPoolItem(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var m = map[int]*caches.MemoryFragmentPoolItem{}
+	m[1] = &caches.MemoryFragmentPoolItem{
+		Refs: 0,
+	}
+	var item = m[1]
+	a.IsTrue(item.Refs == 0)
+	a.IsTrue(atomic.AddInt32(&item.Refs, 1) == 1)
+
+	for _, item2 := range m {
+		t.Log(item2)
+		a.IsTrue(atomic.AddInt32(&item2.Refs, 1) == 2)
+	}
+
+	t.Log(m)
+}
+
+func BenchmarkMemoryFragmentPool_Get_HIT(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var pool = caches.NewMemoryFragmentPool()
+	for i := 0; i < 3000; i++ {
+		ok := pool.Put(make([]byte, 2<<20))
+		if !ok {
+			break
+		}
+	}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			data, ok := pool.Get(2 << 20)
+			if ok {
+				pool.Put(data)
+			}
+		}
+	})
+}
+
+func BenchmarkMemoryFragmentPool_Get_TOTALLY_MISSING(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var pool = caches.NewMemoryFragmentPool()
+	for i := 0; i < 3000; i++ {
+		ok := pool.Put(make([]byte, 2<<20+100))
+		if !ok {
+			break
+		}
+	}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			data, ok := pool.Get(2<<20 + 200)
+			if ok {
+				pool.Put(data)
+			}
+		}
+	})
+}
+
+func BenchmarkMemoryPool_Get_HIT_MISSING(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var pool = caches.NewMemoryFragmentPool()
+	for i := 0; i < 3000; i++ {
+		ok := pool.Put(make([]byte, rands.Int(2, 32)<<20))
+		if !ok {
+			break
+		}
+	}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			data, ok := pool.Get(4 << 20)
+			if ok {
+				pool.Put(data)
+			}
+		}
+	})
+}
+
+func BenchmarkMemoryFragmentPool_GC(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var pool = caches.NewMemoryFragmentPool()
+	pool.SetCapacity(1 << 30)
+	for i := 0; i < 2_000; i++ {
+		pool.Put(make([]byte, 1<<20))
+	}
+
+	var mu = sync.Mutex{}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			mu.Lock()
+			for i := 0; i < 100; i++ {
+				pool.GCNextBucket()
+			}
+			mu.Unlock()
+		}
+	})
+}

internal/caches/open_file.go

@@ -12,14 +12,16 @@ type OpenFile struct {
 	meta    []byte
 	header  []byte
 	version int64
+	size    int64
 }
 
-func NewOpenFile(fp *os.File, meta []byte, header []byte, version int64) *OpenFile {
+func NewOpenFile(fp *os.File, meta []byte, header []byte, version int64, size int64) *OpenFile {
 	return &OpenFile{
 		fp:      fp,
 		meta:    meta,
 		header:  header,
 		version: version,
+		size:    size,
 	}
 }
 

internal/caches/open_file_cache.go

@@ -3,7 +3,9 @@
 package caches
 
 import (
+	"fmt"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
 	"github.com/fsnotify/fsnotify"
 	"github.com/iwind/TeaGo/logs"
@@ -14,26 +16,34 @@ import (
 	"time"
 )
 
+const (
+	maxOpenFileSize = 256 << 20
+)
+
 type OpenFileCache struct {
 	poolMap  map[string]*OpenFilePool // file path => Pool
-	poolList *linkedlist.List
+	poolList *linkedlist.List[*OpenFilePool]
 	watcher  *fsnotify.Watcher
 
 	locker sync.RWMutex
 
-	maxSize int
-	count   int
+	maxCount     int
+	capacitySize int64
+
+	count    int
+	usedSize int64
 }
 
-func NewOpenFileCache(maxSize int) (*OpenFileCache, error) {
-	if maxSize <= 0 {
-		maxSize = 16384
+func NewOpenFileCache(maxCount int) (*OpenFileCache, error) {
+	if maxCount <= 0 {
+		maxCount = 16384
 	}
 
 	var cache = &OpenFileCache{
-		maxSize:  maxSize,
-		poolMap:  map[string]*OpenFilePool{},
-		poolList: linkedlist.NewList(),
+		maxCount:     maxCount,
+		poolMap:      map[string]*OpenFilePool{},
+		poolList:     linkedlist.NewList[*OpenFilePool](),
+		capacitySize: (int64(utils.SystemMemoryGB()) << 30) / 16,
 	}
 
 	watcher, err := fsnotify.NewWatcher()
@@ -58,24 +68,36 @@ func (this *OpenFileCache) Get(filename string) *OpenFile {
 	pool, ok := this.poolMap[filename]
 	this.locker.RUnlock()
 	if ok {
-		file, consumed := pool.Get()
+		file, consumed, consumedSize := pool.Get()
 		if consumed {
 			this.locker.Lock()
 			this.count--
+			this.usedSize -= consumedSize
 
 			// pool如果为空，也不需要从列表中删除，避免put时需要重新创建
 
 			this.locker.Unlock()
 		}
+
 		return file
 	}
 	return nil
 }
 
 func (this *OpenFileCache) Put(filename string, file *OpenFile) {
+	if file.size > maxOpenFileSize {
+		return
+	}
+
 	this.locker.Lock()
 	defer this.locker.Unlock()
 
+	// 如果超过当前容量，则关闭最早的
+	if this.count >= this.maxCount || this.usedSize+file.size >= this.capacitySize {
+		this.consumeHead()
+		return
+	}
+
 	pool, ok := this.poolMap[filename]
 	var success bool
 	if ok {
@@ -92,35 +114,7 @@ func (this *OpenFileCache) Put(filename string, file *OpenFile) {
 	// 检查长度
 	if success {
 		this.count++
-
-		// 如果超过当前容量，则关闭最早的
-		if this.count > this.maxSize {
-			var delta = this.maxSize / 100 // 清理1%
-			if delta == 0 {
-				delta = 1
-			}
-			for i := 0; i < delta; i++ {
-				var head = this.poolList.Head()
-				if head == nil {
-					break
-				}
-
-				var headPool = head.Value.(*OpenFilePool)
-				headFile, consumed := headPool.Get()
-				if consumed {
-					this.count--
-					if headFile != nil {
-						_ = headFile.Close()
-					}
-				}
-
-				if headPool.Len() == 0 {
-					delete(this.poolMap, headPool.filename)
-					this.poolList.Remove(head)
-					_ = this.watcher.Remove(headPool.filename)
-				}
-			}
-		}
+		this.usedSize += file.size
 	}
 }
 
@@ -136,6 +130,7 @@ func (this *OpenFileCache) Close(filename string) {
 		this.poolList.Remove(pool.linkItem)
 		_ = this.watcher.Remove(filename)
 		this.count -= pool.Len()
+		this.usedSize -= pool.usedSize
 	}
 
 	this.locker.Unlock()
@@ -155,18 +150,55 @@ func (this *OpenFileCache) CloseAll() {
 	this.poolList.Reset()
 	_ = this.watcher.Close()
 	this.count = 0
+	this.usedSize = 0
 	this.locker.Unlock()
 }
 
+func (this *OpenFileCache) SetCapacity(capacityBytes int64) {
+	this.capacitySize = capacityBytes
+}
+
 func (this *OpenFileCache) Debug() {
 	var ticker = time.NewTicker(5 * time.Second)
 	goman.New(func() {
 		for range ticker.C {
-			logs.Println("==== " + types.String(this.count) + " ====")
-			this.poolList.Range(func(item *linkedlist.Item) (goNext bool) {
-				logs.Println(filepath.Base(item.Value.(*OpenFilePool).Filename()), item.Value.(*OpenFilePool).Len())
+			logs.Println("==== " + types.String(this.count) + ", " + fmt.Sprintf("%.4fMB", float64(this.usedSize)/(1<<20)) + " ====")
+			this.poolList.Range(func(item *linkedlist.Item[*OpenFilePool]) (goNext bool) {
+				logs.Println(filepath.Base(item.Value.Filename()), item.Value.Len())
 				return true
 			})
 		}
 	})
 }
+
+func (this *OpenFileCache) consumeHead() {
+	var delta = 1
+
+	if this.count > 100 {
+		delta = 2
+	}
+
+	for i := 0; i < delta; i++ {
+		var head = this.poolList.Head()
+		if head == nil {
+			break
+		}
+
+		var headPool = head.Value
+		headFile, consumed, consumedSize := headPool.Get()
+		if consumed {
+			this.count--
+			this.usedSize -= consumedSize
+
+			if headFile != nil {
+				_ = headFile.Close()
+			}
+		}
+
+		if headPool.Len() == 0 {
+			delete(this.poolMap, headPool.filename)
+			this.poolList.Remove(head)
+			_ = this.watcher.Remove(headPool.filename)
+		}
+	}
+}

internal/caches/open_file_cache_test.go

@@ -5,6 +5,7 @@ package caches_test
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/types"
 	"testing"
 	"time"
 )
@@ -15,13 +16,14 @@ func TestNewOpenFileCache_Close(t *testing.T) {
 		t.Fatal(err)
 	}
 	cache.Debug()
-	cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
-	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
-	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
-	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
-	cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
+	cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
+
 	cache.Get("b.txt")
-	cache.Get("d.txt")
+	cache.Get("d.txt") // not exist
 	cache.Close("a.txt")
 
 	if testutils.IsSingleTesting() {
@@ -29,18 +31,39 @@ func TestNewOpenFileCache_Close(t *testing.T) {
 	}
 }
 
+func TestNewOpenFileCache_OverSize(t *testing.T) {
+	cache, err := caches.NewOpenFileCache(1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cache.SetCapacity(1 << 30)
+
+	cache.Debug()
+
+	for i := 0; i < 100; i++ {
+		cache.Put("a"+types.String(i)+".txt", caches.NewOpenFile(nil, nil, nil, 0, 128<<20))
+	}
+
+	if testutils.IsSingleTesting() {
+		time.Sleep(100 * time.Second)
+	}
+}
+
 func TestNewOpenFileCache_CloseAll(t *testing.T) {
 	cache, err := caches.NewOpenFileCache(1024)
 	if err != nil {
 		t.Fatal(err)
 	}
 	cache.Debug()
-	cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
-	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
-	cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0, 1))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1))
+	cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0, 1))
 	cache.Get("b.txt")
 	cache.Get("d.txt")
 	cache.CloseAll()
 
-	time.Sleep(6 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(6 * time.Second)
+	}
 }

internal/caches/open_file_pool.go

@@ -9,10 +9,11 @@ import (
 
 type OpenFilePool struct {
 	c        chan *OpenFile
-	linkItem *linkedlist.Item
+	linkItem *linkedlist.Item[*OpenFilePool]
 	filename string
 	version  int64
 	isClosed bool
+	usedSize int64
 }
 
 func NewOpenFilePool(filename string) *OpenFilePool {
@@ -21,7 +22,7 @@ func NewOpenFilePool(filename string) *OpenFilePool {
 		c:        make(chan *OpenFile, 1024),
 		version:  fasttime.Now().UnixMilli(),
 	}
-	pool.linkItem = linkedlist.NewItem(pool)
+	pool.linkItem = linkedlist.NewItem[*OpenFilePool](pool)
 	return pool
 }
 
@@ -29,27 +30,29 @@ func (this *OpenFilePool) Filename() string {
 	return this.filename
 }
 
-func (this *OpenFilePool) Get() (*OpenFile, bool) {
+func (this *OpenFilePool) Get() (resultFile *OpenFile, consumed bool, consumedSize int64) {
 	// 如果已经关闭，直接返回
 	if this.isClosed {
-		return nil, false
+		return nil, false, 0
 	}
 
 	select {
 	case file := <-this.c:
 		if file != nil {
+			this.usedSize -= file.size
+
 			err := file.SeekStart()
 			if err != nil {
 				_ = file.Close()
-				return nil, true
+				return nil, true, file.size
 			}
 			file.version = this.version
 
-			return file, true
+			return file, true, file.size
 		}
-		return nil, false
+		return nil, false, 0
 	default:
-		return nil, false
+		return nil, false, 0
 	}
 }
 
@@ -69,6 +72,7 @@ func (this *OpenFilePool) Put(file *OpenFile) bool {
 	// 加入Pool
 	select {
 	case this.c <- file:
+		this.usedSize += file.size
 		return true
 	default:
 		// 多余的直接关闭
@@ -81,6 +85,10 @@ func (this *OpenFilePool) Len() int {
 	return len(this.c)
 }
 
+func (this *OpenFilePool) TotalSize() int64 {
+	return this.usedSize
+}
+
 func (this *OpenFilePool) SetClosing() {
 	this.isClosed = true
 }

internal/caches/open_file_pool_test.go

@@ -13,15 +13,15 @@ func TestOpenFilePool_Get(t *testing.T) {
 	var pool = caches.NewOpenFilePool("a")
 	t.Log(pool.Filename())
 	t.Log(pool.Get())
-	t.Log(pool.Put(caches.NewOpenFile(nil, nil, nil, 0)))
+	t.Log(pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1)))
 	t.Log(pool.Get())
 	t.Log(pool.Get())
 }
 
 func TestOpenFilePool_Close(t *testing.T) {
 	var pool = caches.NewOpenFilePool("a")
-	pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
-	pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
+	pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1))
+	pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1))
 	pool.Close()
 }
 
@@ -35,7 +35,7 @@ func TestOpenFilePool_Concurrent(t *testing.T) {
 			defer wg.Done()
 
 			if rands.Int(0, 1) == 1 {
-				pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
+				pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1))
 			}
 			if rands.Int(0, 1) == 0 {
 				pool.Get()

internal/caches/reader_file.go

@@ -366,7 +366,7 @@ func (this *FileReader) Close() error {
 		} else {
 			var cacheMeta = make([]byte, len(this.meta))
 			copy(cacheMeta, this.meta)
-			this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, cacheMeta, this.header, this.LastModified()))
+			this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, cacheMeta, this.header, this.LastModified(), this.bodySize))
 		}
 		return nil
 	}

internal/caches/reader_partial_file.go

@@ -20,7 +20,7 @@ type PartialFileReader struct {
 func NewPartialFileReader(fp *os.File) *PartialFileReader {
 	return &PartialFileReader{
 		FileReader: NewFileReader(fp),
-		rangePath:  partialRangesFilePath(fp.Name()),
+		rangePath:  PartialRangesFilePath(fp.Name()),
 	}
 }
 

internal/caches/storage_file.go

@@ -14,6 +14,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/trackers"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
 	setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/sizes"
@@ -23,6 +24,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	stringutil "github.com/iwind/TeaGo/utils/string"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"github.com/iwind/gosock/pkg/gosock"
 	"github.com/shirou/gopsutil/v3/load"
 	"math"
 	"os"
@@ -58,6 +60,8 @@ const (
 	FileToMemoryMaxSize             = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
 	FileTmpSuffix                   = ".tmp"
 	DefaultMinDiskFreeSpace  uint64 = 5 << 30 // 当前磁盘最小剩余空间
+	DefaultStaleCacheSeconds        = 1200    // 过时缓存留存时间
+	HashKeyLength                   = 32
 )
 
 var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
@@ -85,7 +89,8 @@ type FileStorage struct {
 
 	openFileCache *OpenFileCache
 
-	mainDiskIsFull bool
+	mainDiskIsFull    bool
+	mainDiskTotalSize uint64
 
 	subDirs []*FileDir
 }
@@ -418,6 +423,13 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 		return nil, ErrEntityTooLarge
 	}
 
+	// 检查磁盘是否超出容量
+	// 需要在内存缓存之前执行，避免成功写进到了内存缓存，但无法刷到磁盘
+	var capacityBytes = this.diskCapacityBytes()
+	if capacityBytes > 0 && capacityBytes <= this.TotalDiskSize()+(32<<20 /** 余量 **/) {
+		return nil, NewCapacityError("write file cache failed: over disk size, current: " + types.String(this.TotalDiskSize()) + ", capacity: " + types.String(capacityBytes))
+	}
+
 	// 先尝试内存缓存
 	// 我们限定仅小文件优先存在内存中
 	var maxMemorySize = FileToMemoryMaxSize
@@ -432,7 +444,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 		}
 
 		// 如果队列满了，则等待
-		if err == ErrWritingQueueFull {
+		if errors.Is(err, ErrWritingQueueFull) {
 			return nil, err
 		}
 	}
@@ -461,12 +473,6 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 		}
 	}()
 
-	// 检查是否超出容量
-	var capacityBytes = this.diskCapacityBytes()
-	if capacityBytes > 0 && capacityBytes <= this.TotalDiskSize() {
-		return nil, NewCapacityError("write file cache failed: over disk size, current total size: " + types.String(this.TotalDiskSize()) + " bytes, capacity: " + types.String(capacityBytes))
-	}
-
 	var hash = stringutil.Md5(key)
 
 	dir, diskIsFull := this.subDir(hash)
@@ -556,7 +562,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	if isNewCreated && existsFile {
 		flags |= os.O_TRUNC
 	}
+	fsutils.WriteBegin()
 	writer, err := os.OpenFile(tmpPath, flags, 0666)
+	fsutils.WriteEnd()
 	if err != nil {
 		// TODO 检查在各个系统中的稳定性
 		if os.IsNotExist(err) {
@@ -890,7 +898,7 @@ func (this *FileStorage) Stop() {
 
 // TotalDiskSize 消耗的磁盘尺寸
 func (this *FileStorage) TotalDiskSize() int64 {
-	stat, err := fsutils.StatCache(this.options.Dir)
+	stat, err := fsutils.StatDeviceCache(this.options.Dir)
 	if err == nil {
 		return int64(stat.UsedSize())
 	}
@@ -930,7 +938,7 @@ func (this *FileStorage) keyPath(key string) (hash string, path string, diskIsFu
 
 // 获取Hash对应的文件路径
 func (this *FileStorage) hashPath(hash string) (path string, diskIsFull bool) {
-	if len(hash) != 32 {
+	if len(hash) != HashKeyLength {
 		return "", false
 	}
 	var dir string
@@ -994,28 +1002,31 @@ func (this *FileStorage) initList() error {
 // 清理任务
 // TODO purge每个分区
 func (this *FileStorage) purgeLoop() {
-	// 检查磁盘剩余空间
-	this.checkDiskSpace()
+	// load
+	systemLoad, _ := load.Avg()
+
+	// TODO 计算平均最近每日新增用量
 
 	// 计算是否应该开启LFU清理
 	var capacityBytes = this.diskCapacityBytes()
 	var startLFU = false
+	var requireFullLFU = false // 是否需要完整执行LFU
 	var lfuFreePercent = this.policy.PersistenceLFUFreePercent
 	if lfuFreePercent <= 0 {
 		lfuFreePercent = 5
-	}
 
-	var hasFullDisk = this.mainDiskIsFull
-	if !hasFullDisk {
-		var subDirs = this.subDirs // copy slice
-		for _, subDir := range subDirs {
-			if subDir.IsFull {
-				hasFullDisk = true
-				break
+		if systemLoad == nil || systemLoad.Load5 > 10 {
+			// 2TB级别以上
+			if capacityBytes>>30 > 2000 {
+				lfuFreePercent = 100 /** GB **/ / float32(capacityBytes>>30) * 100 /** % **/
+				if lfuFreePercent > 3 {
+					lfuFreePercent = 3
+				}
 			}
 		}
 	}
 
+	var hasFullDisk = this.hasFullDisk()
 	if hasFullDisk {
 		startLFU = true
 	} else {
@@ -1034,30 +1045,45 @@ func (this *FileStorage) purgeLoop() {
 		var times = 1
 
 		// 空闲时间多清理
-		systemLoad, _ := load.Avg()
 		if systemLoad != nil {
-			if systemLoad.Load5 < 2 {
+			if systemLoad.Load5 < 3 {
 				times = 5
-			} else if systemLoad.Load5 < 3 {
-				times = 3
 			} else if systemLoad.Load5 < 5 {
+				times = 3
+			} else if systemLoad.Load5 < 10 {
 				times = 2
 			}
 		}
 
+		// 高速硬盘多清理
+		if fsutils.DiskIsExtremelyFast() {
+			times *= 8
+		} else if fsutils.DiskIsFast() {
+			times *= 4
+		}
+
 		// 处于LFU阈值时，多清理
 		if startLFU {
-			times = 5
+			times *= 5
 		}
 
 		var purgeCount = this.policy.PersistenceAutoPurgeCount
 		if purgeCount <= 0 {
 			purgeCount = 1000
+
+			if fsutils.DiskIsExtremelyFast() {
+				purgeCount = 4000
+			} else if fsutils.DiskIsFast() {
+				purgeCount = 2000
+			}
 		}
+
 		for i := 0; i < times; i++ {
 			countFound, err := this.list.Purge(purgeCount, func(hash string) error {
 				path, _ := this.hashPath(hash)
+				fsutils.WriteBegin()
 				err := this.removeCacheFile(path)
+				fsutils.WriteEnd()
 				if err != nil && !os.IsNotExist(err) {
 					remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
 				}
@@ -1070,6 +1096,10 @@ func (this *FileStorage) purgeLoop() {
 			}
 
 			if countFound < purgeCount {
+				if i == 0 && startLFU {
+					requireFullLFU = true
+				}
+
 				break
 			}
 
@@ -1079,28 +1109,61 @@ func (this *FileStorage) purgeLoop() {
 
 	// 磁盘空间不足时，清除老旧的缓存
 	if startLFU {
+		var maxCount = 1000
+		var maxLoops = 5
+
+		if fsutils.DiskIsExtremelyFast() {
+			maxCount = 4000
+		} else if fsutils.DiskIsFast() {
+			maxCount = 2000
+		}
+
 		var total, _ = this.list.Count()
 		if total > 0 {
-			var count = types.Int(math.Ceil(float64(total) * float64(lfuFreePercent*2) / 100))
-			if count > 0 {
-				// 限制单次清理的条数，防止占用太多系统资源
-				if count > 2000 {
-					count = 2000
+			for {
+				maxLoops--
+				if maxLoops <= 0 {
+					break
 				}
 
-				remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
+				// 开始清理
+				var count = types.Int(math.Ceil(float64(total) * float64(lfuFreePercent*2) / 100))
+				if count <= 0 {
+					break
+				}
+
+				// 限制单次清理的条数，防止占用太多系统资源
+				if count > maxCount {
+					count = maxCount
+				}
+
+				var before = time.Now()
 				err := this.list.PurgeLFU(count, func(hash string) error {
 					path, _ := this.hashPath(hash)
+					fsutils.WriteBegin()
 					err := this.removeCacheFile(path)
+					fsutils.WriteEnd()
 					if err != nil && !os.IsNotExist(err) {
 						remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
 					}
 
 					return nil
 				})
+
+				var prefix = ""
+				if requireFullLFU {
+					prefix = "fully "
+				}
+				remotelogs.Println("CACHE", prefix+"LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count)+", cost: "+fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000))
+
 				if err != nil {
 					remotelogs.Warn("CACHE", "purge file storage in LFU failed: "+err.Error())
 				}
+
+				// 检查硬盘空间状态
+				if !requireFullLFU && !this.hasFullDisk() {
+					break
+				}
 			}
 		}
 	}
@@ -1108,11 +1171,16 @@ func (this *FileStorage) purgeLoop() {
 
 // 热点数据任务
 func (this *FileStorage) hotLoop() {
-	var memoryStorage = this.memoryStorage
+	var memoryStorage = this.memoryStorage // copy
 	if memoryStorage == nil {
 		return
 	}
 
+	// check memory space size
+	if !memoryStorage.HasFreeSpaceForHotItems() {
+		return
+	}
+
 	this.hotMapLocker.Lock()
 	if len(this.hotMap) == 0 {
 		this.hotMapLocker.Unlock()
@@ -1124,6 +1192,9 @@ func (this *FileStorage) hotLoop() {
 
 	var result = []*HotItem{} // [ {key: ..., hits: ...}, ... ]
 	for _, v := range this.hotMap {
+		if v.Hits <= 1 {
+			continue
+		}
 		result = append(result, v)
 	}
 
@@ -1227,12 +1298,21 @@ func (this *FileStorage) hotLoop() {
 
 func (this *FileStorage) diskCapacityBytes() int64 {
 	var c1 = this.policy.CapacityBytes()
-	if SharedManager.MaxDiskCapacity != nil {
-		var c2 = SharedManager.MaxDiskCapacity.Bytes()
+	var nodeCapacity = SharedManager.MaxDiskCapacity // copy
+	if nodeCapacity != nil {
+		var c2 = nodeCapacity.Bytes()
 		if c2 > 0 {
+			if this.mainDiskTotalSize > 0 && c2 >= int64(this.mainDiskTotalSize) {
+				c2 = int64(this.mainDiskTotalSize) * 95 / 100 // keep 5% free
+			}
 			return c2
 		}
 	}
+
+	if c1 <= 0 || (this.mainDiskTotalSize > 0 && c1 >= int64(this.mainDiskTotalSize)) {
+		c1 = int64(this.mainDiskTotalSize) * 95 / 100 // keep 5% free
+	}
+
 	return c1
 }
 
@@ -1291,19 +1371,9 @@ func (this *FileStorage) increaseHit(key string, hash string, reader Reader) {
 	if rate <= 0 {
 		rate = 1000
 	}
-	if this.lastHotSize == 0 {
-		// 自动降低采样率来增加热点数据的缓存几率
-		rate = rate / 10
-	}
 	if rands.Int(0, rate) == 0 {
 		var memoryStorage = this.memoryStorage
 
-		var hitErr = this.list.IncreaseHit(hash)
-		if hitErr != nil {
-			// 此错误可以忽略
-			remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
-		}
-
 		// 增加到热点
 		// 这里不收录缓存尺寸过大的文件
 		if memoryStorage != nil && reader.BodySize() > 0 && reader.BodySize() < 128*sizes.M {
@@ -1319,6 +1389,15 @@ func (this *FileStorage) increaseHit(key string, hash string, reader Reader) {
 				}
 			}
 			this.hotMapLocker.Unlock()
+
+			// 只有重复点击的才增加点击量
+			if ok {
+				var hitErr = this.list.IncreaseHit(hash)
+				if hitErr != nil {
+					// 此错误可以忽略
+					remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
+				}
+			}
 		}
 	}
 }
@@ -1335,11 +1414,15 @@ func (this *FileStorage) removeCacheFile(path string) error {
 		err = nil
 
 		// 删除Partial相关
-		var partialPath = partialRangesFilePath(path)
+		var partialPath = PartialRangesFilePath(path)
 		if openFileCache != nil {
 			openFileCache.Close(partialPath)
 		}
-		_ = os.Remove(partialPath)
+
+		_, statErr := os.Stat(partialPath)
+		if statErr == nil {
+			_ = os.Remove(partialPath)
+		}
 	}
 	return err
 }
@@ -1418,7 +1501,7 @@ func (this *FileStorage) initOpenFileCache() {
 }
 
 func (this *FileStorage) runMemoryStorageSafety(f func(memoryStorage *MemoryStorage)) {
-	var memoryStorage = this.memoryStorage
+	var memoryStorage = this.memoryStorage // copy
 	if memoryStorage != nil {
 		f(memoryStorage)
 	}
@@ -1434,20 +1517,56 @@ func (this *FileStorage) checkDiskSpace() {
 	}
 
 	if options != nil && len(options.Dir) > 0 {
-		stat, err := fsutils.Stat(options.Dir)
+		stat, err := fsutils.StatDevice(options.Dir)
 		if err == nil {
 			this.mainDiskIsFull = stat.FreeSize() < minFreeSize
+			this.mainDiskTotalSize = stat.TotalSize()
+
+			// check capacity (only on main directory) when node capacity had not been set
+			if !this.mainDiskIsFull {
+				var capacityBytes int64
+				var maxDiskCapacity = SharedManager.MaxDiskCapacity // copy
+				if maxDiskCapacity != nil && maxDiskCapacity.Bytes() > 0 {
+					capacityBytes = SharedManager.MaxDiskCapacity.Bytes()
+				} else {
+					var policy = this.policy // copy
+					if policy != nil {
+						capacityBytes = policy.CapacityBytes() // copy
+					}
+				}
+
+				if capacityBytes > 0 && stat.UsedSize() >= uint64(capacityBytes) {
+					this.mainDiskIsFull = true
+				}
+			}
 		}
 	}
 	var subDirs = this.subDirs // copy slice
 	for _, subDir := range subDirs {
-		stat, err := fsutils.Stat(subDir.Path)
+		stat, err := fsutils.StatDevice(subDir.Path)
 		if err == nil {
 			subDir.IsFull = stat.FreeSize() < minFreeSize
 		}
 	}
 }
 
+// 检查是否有已满的磁盘分区
+func (this *FileStorage) hasFullDisk() bool {
+	this.checkDiskSpace()
+
+	var hasFullDisk = this.mainDiskIsFull
+	if !hasFullDisk {
+		var subDirs = this.subDirs // copy slice
+		for _, subDir := range subDirs {
+			if subDir.IsFull {
+				hasFullDisk = true
+				break
+			}
+		}
+	}
+	return hasFullDisk
+}
+
 // 获取目录
 func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
 	var suffix = "/p" + types.String(this.policy.Id) + "/" + hash[:2] + "/" + hash[2:4]
@@ -1477,6 +1596,121 @@ func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
 	return subDir.Path + suffix, subDir.IsFull
 }
 
+// ScanGarbageCaches 清理目录中“失联”的缓存文件
+// “失联”为不在HashMap中的文件
+func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error) error {
+	if !this.list.(*FileList).HashMapIsLoaded() {
+		return errors.New("cache list is loading")
+	}
+
+	var mainDir = this.options.Dir
+	var allDirs = []string{mainDir}
+	var subDirs = this.subDirs // copy
+	for _, subDir := range subDirs {
+		allDirs = append(allDirs, subDir.Path)
+	}
+
+	var countDirs = 0
+
+	// process progress
+	var progressSock = gosock.NewTmpSock(teaconst.CacheGarbageSockName)
+	_, sockErr := progressSock.SendTimeout(&gosock.Command{Code: "progress", Params: map[string]any{"progress": 0}}, 1*time.Second)
+	var canReportProgress = sockErr == nil
+	var lastProgress float64
+	var countFound = 0
+
+	for _, subDir := range allDirs {
+		var dir0 = subDir + "/p" + types.String(this.policy.Id)
+		dir1Matches, err := filepath.Glob(dir0 + "/*")
+		if err != nil {
+			// ignore error
+			continue
+		}
+
+		for _, dir1 := range dir1Matches {
+			if len(filepath.Base(dir1)) != 2 {
+				continue
+			}
+
+			dir2Matches, err := filepath.Glob(dir1 + "/*")
+			if err != nil {
+				// ignore error
+				continue
+			}
+			for _, dir2 := range dir2Matches {
+				if len(filepath.Base(dir2)) != 2 {
+					continue
+				}
+
+				countDirs++
+
+				// report progress
+				if canReportProgress {
+					var progress = float64(countDirs) / 65536
+					if fmt.Sprintf("%.2f", lastProgress) != fmt.Sprintf("%.2f", progress) {
+						lastProgress = progress
+						_, _ = progressSock.SendTimeout(&gosock.Command{Code: "progress", Params: map[string]any{
+							"progress": progress,
+							"count":    countFound,
+						}}, 100*time.Millisecond)
+					}
+				}
+
+				fileMatches, err := filepath.Glob(dir2 + "/*.cache")
+				if err != nil {
+					// ignore error
+					continue
+				}
+
+				for _, file := range fileMatches {
+					var filename = filepath.Base(file)
+					var hash = strings.TrimSuffix(filename, ".cache")
+					if len(hash) != HashKeyLength {
+						continue
+					}
+
+					isReady, found := this.list.(*FileList).ExistQuick(hash)
+					if !isReady {
+						continue
+					}
+
+					if found {
+						continue
+					}
+
+					// 检查文件正在被写入
+					stat, err := os.Stat(file)
+					if err != nil {
+						continue
+					}
+					if fasttime.Now().Unix()-stat.ModTime().Unix() < 300 /** 5 minutes **/ {
+						continue
+					}
+
+					if fileCallback != nil {
+						countFound++
+						err = fileCallback(file)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// 100% progress
+	if canReportProgress && lastProgress != 1 {
+		_, _ = progressSock.SendTimeout(&gosock.Command{Code: "progress", Params: map[string]any{
+			"progress": 1,
+			"count":    countFound,
+		}}, 100*time.Millisecond)
+	}
+
+	return nil
+}
+
+// 计算字节数字代号
 func (this *FileStorage) charCode(r byte) uint8 {
 	if r >= '0' && r <= '9' {
 		return r - '0'

internal/caches/storage_file_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/logs"
@@ -577,6 +578,29 @@ func TestFileStorage_RemoveCacheFile(t *testing.T) {
 	t.Log(storage.removeCacheFile("/Users/WorkSpace/EdgeProject/EdgeCache/p43/15/7e/157eba0dfc6dfb6fbbf20b1f9e584674.cache"))
 }
 
+func TestFileStorage_ScanGarbageCaches(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
+		Id:      43,
+		Options: map[string]any{"dir": "/Users/WorkSpace/EdgeProject/EdgeCache"},
+	})
+	err := storage.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = storage.ScanGarbageCaches(func(path string) error {
+		t.Log(path, PartialRangesFilePath(path))
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func BenchmarkFileStorage_Read(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 

internal/caches/storage_memory.go

@@ -9,12 +9,9 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
 	setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
-	"github.com/TeaOSLab/EdgeNode/internal/utils/sizes"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/cespare/xxhash"
-	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
-	"github.com/shirou/gopsutil/v3/load"
 	"math"
 	"runtime"
 	"strconv"
@@ -31,6 +28,11 @@ type MemoryItem struct {
 	Status      int
 	IsDone      bool
 	ModifiedAt  int64
+
+	IsPrepared  bool
+	WriteOffset int64
+
+	isReferring bool // if it is referring by other objects
 }
 
 func (this *MemoryItem) IsExpired() bool {
@@ -51,7 +53,7 @@ type MemoryStorage struct {
 
 	purgeTicker *utils.Ticker
 
-	totalSize     int64
+	usedSize      int64
 	writingKeyMap map[string]zero.Zero // key => bool
 
 	ignoreKeys *setutils.FixedSet
@@ -63,7 +65,7 @@ func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage Stora
 
 	if parentStorage != nil {
 		if queueSize <= 0 {
-			queueSize = 2048 + int(policy.CapacityBytes()/sizes.G)*2048
+			queueSize = utils.SystemMemoryGB() * 100_000
 		}
 
 		dirtyChan = make(chan string, queueSize)
@@ -86,10 +88,10 @@ func (this *MemoryStorage) Init() error {
 	_ = this.list.Init()
 
 	this.list.OnAdd(func(item *Item) {
-		atomic.AddInt64(&this.totalSize, item.TotalSize())
+		atomic.AddInt64(&this.usedSize, item.TotalSize())
 	})
 	this.list.OnRemove(func(item *Item) {
-		atomic.AddInt64(&this.totalSize, -item.TotalSize())
+		atomic.AddInt64(&this.usedSize, -item.TotalSize())
 	})
 
 	this.initPurgeTicker()
@@ -122,7 +124,12 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool)
 
 	// read from valuesMap
 	this.locker.RLock()
-	item := this.valuesMap[hash]
+	var item = this.valuesMap[hash]
+
+	if item != nil {
+		item.isReferring = true
+	}
+
 	if item == nil || !item.IsDone {
 		this.locker.RUnlock()
 		return nil, ErrNotFound
@@ -137,17 +144,6 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool)
 		}
 		this.locker.RUnlock()
 
-		// 增加点击量
-		// 1/1000采样
-		// TODO 考虑是否在缓存策略里设置
-		if rands.Int(0, 1000) == 0 {
-			var hitErr = this.list.IncreaseHit(types.String(hash))
-			if hitErr != nil {
-				// 此错误可以忽略
-				remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
-			}
-		}
-
 		return reader, nil
 	}
 	this.locker.RUnlock()
@@ -180,7 +176,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 	if isDirty &&
 		this.parentStorage != nil &&
 		this.dirtyQueueSize > 0 &&
-		len(this.dirtyChan) == this.dirtyQueueSize { // 缓存时间过长
+		len(this.dirtyChan) >= this.dirtyQueueSize-int(fsutils.DiskMaxWrites) /** delta **/ { // 缓存时间过长
 		return nil, ErrWritingQueueFull
 	}
 
@@ -216,13 +212,13 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 		}
 	}
 
-	// 检查是否超出最大值
-	capacityBytes := this.memoryCapacityBytes()
+	// 检查是否超出容量最大值
+	var capacityBytes = this.memoryCapacityBytes()
 	if bodySize < 0 {
 		bodySize = 0
 	}
-	if capacityBytes > 0 && capacityBytes <= this.totalSize+bodySize {
-		return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
+	if capacityBytes > 0 && capacityBytes <= atomic.LoadInt64(&this.usedSize)+bodySize {
+		return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.usedSize, 10) + " bytes")
 	}
 
 	// 先删除
@@ -232,7 +228,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 	}
 
 	isWriting = true
-	return NewMemoryWriter(this, key, expiresAt, status, isDirty, maxSize, func() {
+	return NewMemoryWriter(this, key, expiresAt, status, isDirty, bodySize, maxSize, func(valueItem *MemoryItem) {
 		this.locker.Lock()
 		delete(this.writingKeyMap, key)
 		this.locker.Unlock()
@@ -264,7 +260,7 @@ func (this *MemoryStorage) CleanAll() error {
 	this.locker.Lock()
 	this.valuesMap = map[uint64]*MemoryItem{}
 	_ = this.list.Reset()
-	atomic.StoreInt64(&this.totalSize, 0)
+	atomic.StoreInt64(&this.usedSize, 0)
 	this.locker.Unlock()
 	return nil
 }
@@ -375,6 +371,11 @@ func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePol
 
 // AddToList 将缓存添加到列表
 func (this *MemoryStorage) AddToList(item *Item) {
+	// skip added item
+	if item.MetaSize > 0 {
+		return
+	}
+
 	item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/
 	var hash = types.String(this.hash(item.Key))
 
@@ -392,7 +393,7 @@ func (this *MemoryStorage) TotalDiskSize() int64 {
 
 // TotalMemorySize 内存尺寸
 func (this *MemoryStorage) TotalMemorySize() int64 {
-	return atomic.LoadInt64(&this.totalSize)
+	return atomic.LoadInt64(&this.usedSize)
 }
 
 // IgnoreKey 忽略某个Key，即不缓存某个Key
@@ -405,6 +406,11 @@ func (this *MemoryStorage) CanSendfile() bool {
 	return false
 }
 
+// HasFreeSpaceForHotItems 是否有足够的空间提供给热门内容
+func (this *MemoryStorage) HasFreeSpaceForHotItems() bool {
+	return atomic.LoadInt64(&this.usedSize) < this.memoryCapacityBytes()*3/4
+}
+
 // 计算Key Hash
 func (this *MemoryStorage) hash(key string) uint64 {
 	return xxhash.Sum64String(key)
@@ -412,22 +418,6 @@ func (this *MemoryStorage) hash(key string) uint64 {
 
 // 清理任务
 func (this *MemoryStorage) purgeLoop() {
-	// 计算是否应该开启LFU清理
-	var capacityBytes = this.policy.CapacityBytes()
-	var startLFU = false
-	var usedPercent = float32(this.TotalMemorySize()*100) / float32(capacityBytes)
-	var lfuFreePercent = this.policy.MemoryLFUFreePercent
-	if lfuFreePercent <= 0 {
-		lfuFreePercent = 5
-	}
-	if capacityBytes > 0 {
-		if lfuFreePercent < 100 {
-			if usedPercent >= 100-lfuFreePercent {
-				startLFU = true
-			}
-		}
-	}
-
 	// 清理过期
 	var purgeCount = this.policy.MemoryAutoPurgeCount
 	if purgeCount <= 0 {
@@ -444,6 +434,23 @@ func (this *MemoryStorage) purgeLoop() {
 	})
 
 	// LFU
+	// 计算是否应该开启LFU清理
+	var capacityBytes = this.policy.CapacityBytes()
+	var startLFU = false
+
+	var usedPercent = float32(this.TotalMemorySize()*100) / float32(capacityBytes)
+	var lfuFreePercent = this.policy.MemoryLFUFreePercent
+	if lfuFreePercent <= 0 {
+		lfuFreePercent = 5
+	}
+	if capacityBytes > 0 {
+		if lfuFreePercent < 100 {
+			if usedPercent >= 100-lfuFreePercent {
+				startLFU = true
+			}
+		}
+	}
+
 	if startLFU {
 		var total, _ = this.list.Count()
 		if total > 0 {
@@ -476,33 +483,20 @@ func (this *MemoryStorage) purgeLoop() {
 // 开始Flush任务
 func (this *MemoryStorage) startFlush() {
 	var statCount = 0
-	var writeDelayMS float64 = 0
 
 	for key := range this.dirtyChan {
 		statCount++
 
 		if statCount == 100 {
 			statCount = 0
-
-			// delay some time to reduce load if needed
-			if !fsutils.DiskIsFast() {
-				loadStat, err := load.Avg()
-				if err == nil && loadStat != nil {
-					if loadStat.Load1 > 10 {
-						writeDelayMS = 100
-					} else if loadStat.Load1 > 5 {
-						writeDelayMS = 50
-					} else {
-						writeDelayMS = 0
-					}
-				}
-			}
 		}
 
 		this.flushItem(key)
 
-		if writeDelayMS > 0 {
-			time.Sleep(time.Duration(writeDelayMS) * time.Millisecond)
+		if fsutils.IsInExtremelyHighLoad {
+			time.Sleep(1 * time.Second)
+		} else if fsutils.IsInHighLoad {
+			time.Sleep(100 * time.Millisecond)
 		}
 	}
 }
@@ -518,9 +512,20 @@ func (this *MemoryStorage) flushItem(key string) {
 	item, ok := this.valuesMap[hash]
 	this.locker.RUnlock()
 
+	// 从内存中移除，并确保无论如何都会执行
+	defer func() {
+		_ = this.Delete(key)
+
+		// 重用内存，前提是确保内存不再被引用
+		if ok && item.IsDone && !item.isReferring && len(item.BodyValue) > 0 {
+			SharedFragmentMemoryPool.Put(item.BodyValue)
+		}
+	}()
+
 	if !ok {
 		return
 	}
+
 	if !item.IsDone {
 		remotelogs.Error("CACHE", "flush items failed: open writer failed: item has not been done")
 		return
@@ -529,6 +534,16 @@ func (this *MemoryStorage) flushItem(key string) {
 		return
 	}
 
+	// 检查是否在列表中，防止未加入列表时就开始flush
+	isInList, err := this.list.Exist(types.String(hash))
+	if err != nil {
+		remotelogs.Error("CACHE", "flush items failed: "+err.Error())
+		return
+	}
+	if !isInList {
+		time.Sleep(1 * time.Second)
+	}
+
 	writer, err := this.parentStorage.OpenFlushWriter(key, item.ExpiresAt, item.Status, len(item.HeaderValue), int64(len(item.BodyValue)))
 	if err != nil {
 		if !CanIgnoreErr(err) {
@@ -566,26 +581,37 @@ func (this *MemoryStorage) flushItem(key string) {
 		HeaderSize: writer.HeaderSize(),
 		BodySize:   writer.BodySize(),
 	})
-
-	// 从内存中移除
-	_ = this.Delete(key)
 }
 
 func (this *MemoryStorage) memoryCapacityBytes() int64 {
+	var maxSystemBytes = SharedManager.MaxSystemMemoryBytesPerStorage()
 	if this.policy == nil {
-		return 0
-	}
-	c1 := int64(0)
-	if this.policy.Capacity != nil {
-		c1 = this.policy.Capacity.Bytes()
+		return maxSystemBytes
 	}
+
 	if SharedManager.MaxMemoryCapacity != nil {
-		c2 := SharedManager.MaxMemoryCapacity.Bytes()
-		if c2 > 0 {
-			return c2
+		var capacityBytes = SharedManager.MaxMemoryCapacity.Bytes()
+		if capacityBytes > 0 {
+			if capacityBytes > maxSystemBytes {
+				return maxSystemBytes
+			}
+
+			return capacityBytes
 		}
 	}
-	return c1
+
+	var capacity = this.policy.Capacity // copy
+	if capacity != nil {
+		var capacityBytes = capacity.Bytes()
+		if capacityBytes > 0 {
+			if capacityBytes > maxSystemBytes {
+				return maxSystemBytes
+			}
+			return capacityBytes
+		}
+	}
+
+	return maxSystemBytes
 }
 
 func (this *MemoryStorage) deleteWithoutLocker(key string) error {

internal/caches/utils_partial.go

@@ -4,8 +4,8 @@ package caches
 
 import "strings"
 
-// 获取 ranges 文件路径
-func partialRangesFilePath(path string) string {
+// PartialRangesFilePath 获取 ranges 文件路径
+func PartialRangesFilePath(path string) string {
 	// ranges路径
 	var dotIndex = strings.LastIndex(path, ".")
 	var rangePath string

internal/caches/writer_memory.go

@@ -2,9 +2,9 @@ package caches
 
 import (
 	"errors"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/cespare/xxhash"
 	"sync"
-	"time"
 )
 
 type MemoryWriter struct {
@@ -16,29 +16,49 @@ type MemoryWriter struct {
 	bodySize   int64
 	status     int
 	isDirty    bool
-	maxSize    int64
+
+	expectedBodySize int64
+	maxSize          int64
 
 	hash    uint64
 	item    *MemoryItem
-	endFunc func()
+	endFunc func(valueItem *MemoryItem)
 	once    sync.Once
 }
 
-func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, maxSize int64, endFunc func()) *MemoryWriter {
-	w := &MemoryWriter{
-		storage:   memoryStorage,
-		key:       key,
-		expiredAt: expiredAt,
-		item: &MemoryItem{
-			ExpiresAt:  expiredAt,
-			ModifiedAt: time.Now().Unix(),
-			Status:     status,
-		},
-		status:  status,
-		isDirty: isDirty,
-		maxSize: maxSize,
-		endFunc: endFunc,
+func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, expectedBodySize int64, maxSize int64, endFunc func(valueItem *MemoryItem)) *MemoryWriter {
+	var valueItem = &MemoryItem{
+		ExpiresAt:  expiredAt,
+		ModifiedAt: fasttime.Now().Unix(),
+		Status:     status,
 	}
+	if expectedBodySize > 0 && expectedBodySize <= maxMemoryFragmentPoolItemSize {
+		bodyBytes, ok := SharedFragmentMemoryPool.Get(expectedBodySize) // try to reuse memory
+		if ok {
+			valueItem.BodyValue = bodyBytes
+			valueItem.IsPrepared = true
+		} else {
+			if expectedBodySize <= (16 << 20) {
+				var allocSize = (expectedBodySize/16384 + 1) * 16384
+				valueItem.BodyValue = make([]byte, allocSize)[:expectedBodySize]
+				valueItem.IsPrepared = true
+
+				SharedFragmentMemoryPool.IncreaseNew()
+			}
+		}
+	}
+	var w = &MemoryWriter{
+		storage:          memoryStorage,
+		key:              key,
+		expiredAt:        expiredAt,
+		item:             valueItem,
+		status:           status,
+		isDirty:          isDirty,
+		expectedBodySize: expectedBodySize,
+		maxSize:          maxSize,
+		endFunc:          endFunc,
+	}
+
 	w.hash = w.calculateHash(key)
 
 	return w
@@ -53,17 +73,32 @@ func (this *MemoryWriter) WriteHeader(data []byte) (n int, err error) {
 
 // Write 写入数据
 func (this *MemoryWriter) Write(data []byte) (n int, err error) {
-	this.bodySize += int64(len(data))
-	this.item.BodyValue = append(this.item.BodyValue, data...)
+	var l = len(data)
+	if l == 0 {
+		return
+	}
+
+	if this.item.IsPrepared {
+		if this.item.WriteOffset+int64(l) > this.expectedBodySize {
+			err = ErrWritingUnavailable
+			return
+		}
+		copy(this.item.BodyValue[this.item.WriteOffset:], data)
+		this.item.WriteOffset += int64(l)
+	} else {
+		this.item.BodyValue = append(this.item.BodyValue, data...)
+	}
+
+	this.bodySize += int64(l)
 
 	// 检查尺寸
 	if this.maxSize > 0 && this.bodySize > this.maxSize {
 		err = ErrEntityTooLarge
 		this.storage.IgnoreKey(this.key, this.maxSize)
-		return len(data), err
+		return l, err
 	}
 
-	return len(data), nil
+	return l, nil
 }
 
 // WriteAt 在指定位置写入数据
@@ -87,7 +122,8 @@ func (this *MemoryWriter) BodySize() int64 {
 func (this *MemoryWriter) Close() error {
 	// 需要在Locker之外
 	defer this.once.Do(func() {
-		this.endFunc()
+		this.endFunc(this.item)
+		this.item = nil // free memory
 	})
 
 	if this.item == nil {
@@ -96,30 +132,48 @@ func (this *MemoryWriter) Close() error {
 
 	this.storage.locker.Lock()
 	this.item.IsDone = true
-	this.storage.valuesMap[this.hash] = this.item
+	var err error
 	if this.isDirty {
 		if this.storage.parentStorage != nil {
+			this.storage.valuesMap[this.hash] = this.item
+
 			select {
 			case this.storage.dirtyChan <- this.key:
 			default:
+				// remove from values map
+				delete(this.storage.valuesMap, this.hash)
 
+				err = ErrWritingQueueFull
 			}
+		} else {
+			this.storage.valuesMap[this.hash] = this.item
 		}
+	} else {
+		this.storage.valuesMap[this.hash] = this.item
 	}
+
 	this.storage.locker.Unlock()
 
-	return nil
+	return err
 }
 
 // Discard 丢弃
 func (this *MemoryWriter) Discard() error {
 	// 需要在Locker之外
 	defer this.once.Do(func() {
-		this.endFunc()
+		this.endFunc(this.item)
+		this.item = nil // free memory
 	})
 
 	this.storage.locker.Lock()
 	delete(this.storage.valuesMap, this.hash)
+
+	if this.item != nil &&
+		!this.item.isReferring &&
+		cap(this.item.BodyValue) >= minMemoryFragmentPoolItemSize {
+		SharedFragmentMemoryPool.Put(this.item.BodyValue)
+	}
+
 	this.storage.locker.Unlock()
 	return nil
 }

internal/caches/writer_partial_file.go

@@ -43,7 +43,7 @@ func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, metaH
 		isPartial:      isPartial,
 		bodyOffset:     bodyOffset,
 		ranges:         ranges,
-		rangePath:      partialRangesFilePath(rawWriter.Name()),
+		rangePath:      PartialRangesFilePath(rawWriter.Name()),
 		metaHeaderSize: metaHeaderSize,
 		metaBodySize:   metaBodySize,
 	}

internal/configs/api_config.go

@@ -69,6 +69,7 @@ func LoadAPIConfig() (*APIConfig, error) {
 
 		// 自动生成新的配置文件
 		if filename == oldConfigFileName {
+			config.OldRPC.Endpoints = nil
 			_ = config.WriteFile(Tea.ConfigFile(ConfigFileName))
 		}
 

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "1.2.8"
+	Version = "1.3.0"
 
 	ProductName = "Edge Node"
 	ProcessName = "edge-node"
@@ -14,5 +14,6 @@ const (
 	// SystemdServiceName systemd
 	SystemdServiceName = "edge-node"
 
-	AccessLogSockName = "edge-node.accesslog.sock"
+	AccessLogSockName    = "edge-node.accesslog"
+	CacheGarbageSockName = "edge-node.cache.garbage"
 )

internal/iplibrary/ip_list.go

@@ -14,6 +14,8 @@ var GlobalWhiteIPList = NewIPList()
 // IPList IP名单
 // TODO IP名单可以分片关闭，这样让每一片的数据量减少，查询更快
 type IPList struct {
+	isDeleted bool
+
 	itemsMap    map[uint64]*IPItem // id => item
 	sortedItems []*IPItem
 	allItemsMap map[uint64]*IPItem // id => item
@@ -38,11 +40,19 @@ func NewIPList() *IPList {
 }
 
 func (this *IPList) Add(item *IPItem) {
+	if this.isDeleted {
+		return
+	}
+
 	this.addItem(item, true)
 }
 
 // AddDelay 延迟添加，需要手工调用Sort()函数
 func (this *IPList) AddDelay(item *IPItem) {
+	if this.isDeleted {
+		return
+	}
+
 	this.addItem(item, false)
 }
 
@@ -60,6 +70,10 @@ func (this *IPList) Delete(itemId uint64) {
 
 // Contains 判断是否包含某个IP
 func (this *IPList) Contains(ip uint64) bool {
+	if this.isDeleted {
+		return false
+	}
+
 	this.locker.RLock()
 	if len(this.allItemsMap) > 0 {
 		this.locker.RUnlock()
@@ -75,6 +89,10 @@ func (this *IPList) Contains(ip uint64) bool {
 
 // ContainsExpires 判断是否包含某个IP
 func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) {
+	if this.isDeleted {
+		return
+	}
+
 	this.locker.RLock()
 	if len(this.allItemsMap) > 0 {
 		this.locker.RUnlock()
@@ -94,6 +112,10 @@ func (this *IPList) ContainsExpires(ip uint64) (expiresAt int64, ok bool) {
 
 // ContainsIPStrings 是否包含一组IP中的任意一个，并返回匹配的第一个Item
 func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found bool) {
+	if this.isDeleted {
+		return
+	}
+
 	if len(ipStrings) == 0 {
 		return
 	}
@@ -125,6 +147,10 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
 	return
 }
 
+func (this *IPList) SetDeleted() {
+	this.isDeleted = true
+}
+
 func (this *IPList) addItem(item *IPItem, sortable bool) {
 	if item == nil {
 		return

internal/iplibrary/ip_list_db.go

@@ -60,7 +60,7 @@ func (this *IPListDB) init() error {
 
 	var path = this.dir + "/ip_list.db"
 
-	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
+	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
 	if err != nil {
 		return err
 	}

internal/iplibrary/ip_list_db_test.go

@@ -22,10 +22,10 @@ func TestIPListDB_AddItem(t *testing.T) {
 		IpFrom:                        "192.168.1.101",
 		IpTo:                          "",
 		Version:                       1024,
-		ExpiredAt:                     time.Now().Unix(),
+		ExpiredAt:                     time.Now().Unix() + 3600,
 		Reason:                        "",
 		ListId:                        2,
-		IsDeleted:                     true,
+		IsDeleted:                     false,
 		Type:                          "ipv4",
 		EventLevel:                    "error",
 		ListType:                      "black",

internal/iplibrary/manager_ip_list.go

@@ -229,6 +229,11 @@ func (this *IPListManager) DeleteExpiredItems() {
 func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 	var changedLists = map[*IPList]zero.Zero{}
 	for _, item := range items {
+		// 调试
+		if Tea.IsTesting() {
+			this.debugItem(item)
+		}
+
 		var list *IPList
 		// TODO 实现节点专有List
 		if item.ServerId > 0 { // 服务专有List
@@ -300,3 +305,12 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 		}
 	}
 }
+
+// 调试IP信息
+func (this *IPListManager) debugItem(item *pb.IPItem) {
+	if item.IsDeleted {
+		remotelogs.Debug("IP_ITEM_DEBUG", "delete '"+item.IpFrom+"'")
+	} else {
+		remotelogs.Debug("IP_ITEM_DEBUG", "add '"+item.IpFrom+"'")
+	}
+}

internal/metrics/manager.go

@@ -26,9 +26,9 @@ func init() {
 type Manager struct {
 	isQuiting bool
 
-	tasks         map[int64]*Task    // itemId => *Task
-	categoryTasks map[string][]*Task // category => []*Task
-	locker        sync.RWMutex
+	taskMap         map[int64]*Task    // itemId => *Task
+	categoryTaskMap map[string][]*Task // category => []*Task
+	locker          sync.RWMutex
 
 	hasHTTPMetrics bool
 	hasTCPMetrics  bool
@@ -37,8 +37,8 @@ type Manager struct {
 
 func NewManager() *Manager {
 	return &Manager{
-		tasks:         map[int64]*Task{},
-		categoryTasks: map[string][]*Task{},
+		taskMap:         map[int64]*Task{},
+		categoryTaskMap: map[string][]*Task{},
 	}
 }
 
@@ -56,7 +56,7 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
 	}
 
 	// 停用以前的 或 修改现在的
-	for itemId, task := range this.tasks {
+	for itemId, task := range this.taskMap {
 		newItem, ok := newMap[itemId]
 		if !ok || !newItem.IsOn { // 停用以前的
 			remotelogs.Println("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"'")
@@ -64,7 +64,7 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
 			if err != nil {
 				remotelogs.Error("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"' failed: "+err.Error())
 			}
-			delete(this.tasks, itemId)
+			delete(this.taskMap, itemId)
 		} else { // 更新已存在的
 			if newItem.Version != task.item.Version {
 				remotelogs.Println("METRIC_MANAGER", "update task '"+strconv.FormatInt(itemId, 10)+"'")
@@ -78,7 +78,7 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
 		if !newItem.IsOn {
 			continue
 		}
-		_, ok := this.tasks[newItem.Id]
+		_, ok := this.taskMap[newItem.Id]
 		if !ok {
 			remotelogs.Println("METRIC_MANAGER", "start task '"+strconv.FormatInt(newItem.Id, 10)+"'")
 			task := NewTask(newItem)
@@ -92,7 +92,7 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
 				remotelogs.Error("METRIC_MANAGER", "start task failed: "+err.Error())
 				continue
 			}
-			this.tasks[newItem.Id] = task
+			this.taskMap[newItem.Id] = task
 		}
 	}
 
@@ -100,11 +100,11 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
 	this.hasHTTPMetrics = false
 	this.hasTCPMetrics = false
 	this.hasUDPMetrics = false
-	this.categoryTasks = map[string][]*Task{}
-	for _, task := range this.tasks {
-		tasks := this.categoryTasks[task.item.Category]
+	this.categoryTaskMap = map[string][]*Task{}
+	for _, task := range this.taskMap {
+		var tasks = this.categoryTaskMap[task.item.Category]
 		tasks = append(tasks, task)
-		this.categoryTasks[task.item.Category] = tasks
+		this.categoryTaskMap[task.item.Category] = tasks
 
 		switch task.item.Category {
 		case serverconfigs.MetricItemCategoryHTTP:
@@ -124,10 +124,12 @@ func (this *Manager) Add(obj MetricInterface) {
 	}
 
 	this.locker.RLock()
-	for _, task := range this.categoryTasks[obj.MetricCategory()] {
+	var tasks = this.categoryTaskMap[obj.MetricCategory()]
+	this.locker.RUnlock()
+
+	for _, task := range tasks {
 		task.Add(obj)
 	}
-	this.locker.RUnlock()
 }
 
 func (this *Manager) HasHTTPMetrics() bool {
@@ -149,9 +151,9 @@ func (this *Manager) Quit() {
 	remotelogs.Println("METRIC_MANAGER", "quit")
 
 	this.locker.Lock()
-	for _, task := range this.tasks {
+	for _, task := range this.taskMap {
 		_ = task.Stop()
 	}
-	this.tasks = map[int64]*Task{}
+	this.taskMap = map[int64]*Task{}
 	this.locker.Unlock()
 }

internal/metrics/manager_test.go

@@ -11,7 +11,7 @@ func TestNewManager(t *testing.T) {
 	var manager = NewManager()
 	{
 		manager.Update([]*serverconfigs.MetricItemConfig{})
-		for _, task := range manager.tasks {
+		for _, task := range manager.taskMap {
 			t.Log(task.item.Id)
 		}
 	}
@@ -28,7 +28,7 @@ func TestNewManager(t *testing.T) {
 				Id: 3,
 			},
 		})
-		for _, task := range manager.tasks {
+		for _, task := range manager.taskMap {
 			t.Log("task:", task.item.Id)
 		}
 	}
@@ -43,7 +43,7 @@ func TestNewManager(t *testing.T) {
 				Id: 2,
 			},
 		})
-		for _, task := range manager.tasks {
+		for _, task := range manager.taskMap {
 			t.Log("task:", task.item.Id)
 		}
 	}
@@ -56,7 +56,7 @@ func TestNewManager(t *testing.T) {
 				Version: 1,
 			},
 		})
-		for _, task := range manager.tasks {
+		for _, task := range manager.taskMap {
 			t.Log("task:", task.item.Id)
 		}
 	}

internal/metrics/stat.go

@@ -17,6 +17,6 @@ type Stat struct {
 }
 
 func SumStat(serverId int64, keys []string, time string, version int32, itemId int64) string {
-	keysData := strings.Join(keys, "$EDGE$")
+	var keysData = strings.Join(keys, "$EDGE$")
 	return strconv.FormatUint(fnv.HashString(strconv.FormatInt(serverId, 10)+"@"+keysData+"@"+time+"@"+strconv.Itoa(int(version))+"@"+strconv.FormatInt(itemId, 10)), 10)
 }

internal/metrics/sum_test.go

@@ -0,0 +1,20 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package metrics_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/metrics"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"runtime"
+	"testing"
+)
+
+func BenchmarkSumStat(b *testing.B) {
+	runtime.GOMAXPROCS(2)
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			metrics.SumStat(1, []string{"1.2.3.4"}, timeutil.Format("Ymd"), 1, 1)
+		}
+	})
+}

internal/metrics/task.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -59,7 +60,7 @@ type Task struct {
 	serverIdMapLocker sync.Mutex
 
 	statsMap    map[string]*Stat // 待写入队列，hash => *Stat
-	statsLocker sync.Mutex
+	statsLocker sync.RWMutex
 	statsTicker *utils.Ticker
 }
 
@@ -90,7 +91,7 @@ func (this *Task) Init() error {
 
 	var path = dir + "/metric." + types.String(this.item.Id) + ".db"
 
-	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
+	db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
 	if err != nil {
 		return err
 	}
@@ -237,7 +238,7 @@ func (this *Task) Add(obj MetricInterface) {
 
 	var keys = []string{}
 	for _, key := range this.item.Keys {
-		k := obj.MetricKey(key)
+		var k = obj.MetricKey(key)
 
 		// 忽略499状态
 		if key == "${status}" && k == "499" {
@@ -253,14 +254,19 @@ func (this *Task) Add(obj MetricInterface) {
 	}
 
 	var hash = SumStat(obj.MetricServerId(), keys, this.item.CurrentTime(), this.item.Version, this.item.Id)
-	this.statsLocker.Lock()
+	var countItems int
+	this.statsLocker.RLock()
 	oldStat, ok := this.statsMap[hash]
+	if !ok {
+		countItems = len(this.statsMap)
+	}
+	this.statsLocker.RUnlock()
 	if ok {
-		oldStat.Value += v
-		oldStat.Hash = hash
+		atomic.AddInt64(&oldStat.Value, 1)
 	} else {
 		// 防止过载
-		if len(this.statsMap) < MaxQueueSize {
+		if countItems < MaxQueueSize {
+			this.statsLocker.Lock()
 			this.statsMap[hash] = &Stat{
 				ServerId: obj.MetricServerId(),
 				Keys:     keys,
@@ -268,9 +274,9 @@ func (this *Task) Add(obj MetricInterface) {
 				Time:     this.item.CurrentTime(),
 				Hash:     hash,
 			}
+			this.statsLocker.Unlock()
 		}
 	}
-	this.statsLocker.Unlock()
 }
 
 // Stop 停止任务

internal/metrics/task_test.go

@@ -4,11 +4,15 @@ package metrics_test
 
 import (
 	"fmt"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/metrics"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/rands"
+	"log"
+	"runtime"
+	"sync"
 	"testing"
 	"time"
 )
@@ -213,3 +217,65 @@ func TestTask_Upload(t *testing.T) {
 	}
 	t.Log("ok")
 }
+
+var testingTask *metrics.Task
+var testingTaskInitOnce = &sync.Once{}
+
+func initTestingTask() {
+	testingTask = metrics.NewTask(&serverconfigs.MetricItemConfig{
+		Id:         1,
+		IsOn:       false,
+		Category:   "tcp",
+		Period:     1,
+		PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
+		Keys:       []string{"${remoteAddr}"},
+		Value:      "${countRequest}",
+	})
+
+	err := testingTask.Init()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = testingTask.Start()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func BenchmarkTask_Add(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	testingTaskInitOnce.Do(func() {
+		initTestingTask()
+	})
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			testingTask.Add(&taskRequest{})
+		}
+	})
+}
+
+type taskRequest struct {
+}
+
+func (this *taskRequest) MetricKey(key string) string {
+	return configutils.ParseVariables(key, func(varName string) (value string) {
+		return "1.2.3.4"
+	})
+}
+
+func (this *taskRequest) MetricValue(value string) (result int64, ok bool) {
+	return 1, true
+}
+
+func (this *taskRequest) MetricServerId() int64 {
+	return 1
+}
+
+func (this *taskRequest) MetricCategory() string {
+	return "http"
+}

internal/nodes/client_conn.go

@@ -24,8 +24,6 @@ import (
 	"time"
 )
 
-var synFloodCounter = counters.NewCounter().WithGC()
-
 // ClientConn 客户端连接
 type ClientConn struct {
 	BaseClientConn
@@ -198,9 +196,9 @@ func (this *ClientConn) Write(b []byte) (n int, err error) {
 
 				var cost = time.Since(before).Seconds()
 				if cost > 1 {
-					stats.SharedBandwidthStatManager.AddBandwidth(this.userId, this.serverId, int64(float64(n)/cost), int64(n))
+					stats.SharedBandwidthStatManager.AddBandwidth(this.userId, this.userPlanId, this.serverId, int64(float64(n)/cost), int64(n))
 				} else {
-					stats.SharedBandwidthStatManager.AddBandwidth(this.userId, this.serverId, int64(n), int64(n))
+					stats.SharedBandwidthStatManager.AddBandwidth(this.userId, this.userPlanId, this.serverId, int64(n), int64(n))
 				}
 			}
 		}
@@ -292,13 +290,13 @@ func (this *ClientConn) LastErr() error {
 }
 
 func (this *ClientConn) resetSYNFlood() {
-	synFloodCounter.ResetKey("SYN_FLOOD:" + this.RawIP())
+	counters.SharedCounter.ResetKey("SYN_FLOOD:" + this.RawIP())
 }
 
 func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloodConfig) {
 	var ip = this.RawIP()
 	if len(ip) > 0 && !iplibrary.IsInWhiteList(ip) && (!synFloodConfig.IgnoreLocal || !utils.IsLocalIP(ip)) {
-		var result = synFloodCounter.IncreaseKey("SYN_FLOOD:"+ip, 60)
+		var result = counters.SharedCounter.IncreaseKey("SYN_FLOOD:"+ip, 60)
 		var minAttempts = synFloodConfig.MinAttempts
 		if minAttempts < 5 {
 			minAttempts = 5
@@ -307,7 +305,7 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
 			// 非TLS，设置为两倍，防止误封
 			minAttempts = 2 * minAttempts
 		}
-		if result >= types.Uint64(minAttempts) {
+		if result >= types.Uint32(minAttempts) {
 			var timeout = synFloodConfig.TimeoutSeconds
 			if timeout <= 0 {
 				timeout = 600

internal/nodes/client_conn_base.go

@@ -16,6 +16,7 @@ type BaseClientConn struct {
 
 	isBound    bool
 	userId     int64
+	userPlanId int64
 	serverId   int64
 	remoteAddr string
 	hasLimit   bool
@@ -106,11 +107,31 @@ func (this *BaseClientConn) SetUserId(userId int64) {
 	}
 }
 
+func (this *BaseClientConn) SetUserPlanId(userPlanId int64) {
+	this.userPlanId = userPlanId
+
+	// 设置包装前连接
+	switch conn := this.rawConn.(type) {
+	case *tls.Conn:
+		nativeConn, ok := conn.NetConn().(ClientConnInterface)
+		if ok {
+			nativeConn.SetUserPlanId(userPlanId)
+		}
+	case *ClientConn:
+		conn.SetUserPlanId(userPlanId)
+	}
+}
+
 // UserId 获取当前连接所属服务的用户ID
 func (this *BaseClientConn) UserId() int64 {
 	return this.userId
 }
 
+// UserPlanId 用户套餐ID
+func (this *BaseClientConn) UserPlanId() int64 {
+	return this.userPlanId
+}
+
 // RawIP 原本IP
 func (this *BaseClientConn) RawIP() string {
 	if len(this.rawIP) > 0 {

internal/nodes/client_conn_interface.go

@@ -18,9 +18,12 @@ type ClientConnInterface interface {
 	// SetServerId 设置服务ID
 	SetServerId(serverId int64) (goNext bool)
 
-	// SetUserId 设置所属服务的用户ID
+	// SetUserId 设置所属网站的用户ID
 	SetUserId(userId int64)
 
+	// SetUserPlanId 设置
+	SetUserPlanId(userPlanId int64)
+
 	// UserId 获取当前连接所属服务的用户ID
 	UserId() int64
 

internal/nodes/http_request.go

@@ -38,15 +38,16 @@ type HTTPRequest struct {
 	requestId string
 
 	// 外部参数
-	RawReq     *http.Request
-	RawWriter  http.ResponseWriter
-	ReqServer  *serverconfigs.ServerConfig
-	ReqHost    string // 请求的Host
-	ServerName string // 实际匹配到的Host
-	ServerAddr string // 实际启动的服务器监听地址
-	IsHTTP     bool
-	IsHTTPS    bool
-	IsHTTP3    bool
+	RawReq        *http.Request
+	RawWriter     http.ResponseWriter
+	ReqServer     *serverconfigs.ServerConfig
+	ReqHost       string // 请求的Host
+	ServerName    string // 实际匹配到的Host
+	ServerAddr    string // 实际启动的服务器监听地址
+	IsHTTP        bool
+	IsHTTPS       bool
+	IsHTTP3       bool
+	isHealthCheck bool
 
 	// 共享参数
 	nodeConfig *nodeconfigs.NodeConfig
@@ -168,10 +169,9 @@ func (this *HTTPRequest) Do() {
 	}
 
 	// 处理健康检查
-	var isHealthCheck = false
 	var healthCheckKey = this.RawReq.Header.Get(serverconfigs.HealthCheckHeaderName)
 	if len(healthCheckKey) > 0 {
-		if this.doHealthCheck(healthCheckKey, &isHealthCheck) {
+		if this.doHealthCheck(healthCheckKey, &this.isHealthCheck) {
 			this.doEnd()
 			return
 		}
@@ -198,14 +198,22 @@ func (this *HTTPRequest) Do() {
 		}
 
 		// 流量限制
-		if this.ReqServer.TrafficLimit != nil && this.ReqServer.TrafficLimit.IsOn && !this.ReqServer.TrafficLimit.IsEmpty() && this.ReqServer.TrafficLimitStatus != nil && this.ReqServer.TrafficLimitStatus.IsValid() {
+		if this.ReqServer.TrafficLimitStatus != nil && this.ReqServer.TrafficLimitStatus.IsValid() {
 			this.doTrafficLimit()
 			this.doEnd()
 			return
 		}
 
+		// WAF
+		if this.web.FirewallRef != nil && this.web.FirewallRef.IsOn {
+			if this.doWAFRequest() {
+				this.doEnd()
+				return
+			}
+		}
+
 		// UAM
-		if !isHealthCheck {
+		if !this.isHealthCheck {
 			if this.web.UAM != nil {
 				if this.web.UAM.IsOn {
 					if this.doUAM() {
@@ -223,7 +231,7 @@ func (this *HTTPRequest) Do() {
 		}
 
 		// CC
-		if !isHealthCheck {
+		if !this.isHealthCheck {
 			if this.web.CC != nil {
 				if this.web.CC.IsOn {
 					if this.doCC() {
@@ -234,14 +242,6 @@ func (this *HTTPRequest) Do() {
 			}
 		}
 
-		// WAF
-		if this.web.FirewallRef != nil && this.web.FirewallRef.IsOn {
-			if this.doWAFRequest() {
-				this.doEnd()
-				return
-			}
-		}
-
 		// 防盗链
 		if !this.isSubRequest && this.web.Referers != nil && this.web.Referers.IsOn {
 			if this.doCheckReferers() {
@@ -388,7 +388,7 @@ func (this *HTTPRequest) doEnd() {
 
 	// 流量统计
 	// TODO 增加是否开启开关
-	if this.ReqServer != nil && this.ReqServer.Id > 0 {
+	if this.ReqServer != nil && this.ReqServer.Id > 0 && !this.isHealthCheck /** 健康检查时不统计 **/ {
 		var totalBytes int64 = 0
 
 		var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
@@ -854,6 +854,24 @@ func (this *HTTPRequest) Format(source string) string {
 			return this.requestHeadersString()
 		case "serverName":
 			return this.ServerName
+		case "serverAddr":
+			var nodeConfig = this.nodeConfig
+			if nodeConfig != nil && nodeConfig.GlobalServerConfig != nil && nodeConfig.GlobalServerConfig.HTTPAll.EnableServerAddrVariable {
+				if len(this.requestRemoteAddrs()) > 1 {
+					return "" // hidden for security
+				}
+				var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
+				if requestConn != nil {
+					conn, ok := requestConn.(net.Conn)
+					if ok {
+						host, _, _ := net.SplitHostPort(conn.LocalAddr().String())
+						if len(host) > 0 {
+							return host
+						}
+					}
+				}
+			}
+			return ""
 		case "serverPort":
 			return strconv.Itoa(this.requestServerPort())
 		case "hostname":
@@ -1153,10 +1171,32 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
 		this.web.RemoteAddr != nil &&
 		this.web.RemoteAddr.IsOn &&
 		!this.web.RemoteAddr.IsEmpty() {
-		var remoteAddr = this.Format(this.web.RemoteAddr.Value)
-		if net.ParseIP(remoteAddr) != nil {
-			this.remoteAddr = remoteAddr
-			return remoteAddr
+		if this.web.RemoteAddr.HasValues() { // multiple values
+			for _, value := range this.web.RemoteAddr.Values() {
+				var remoteAddr = this.Format(value)
+				if len(remoteAddr) > 0 && net.ParseIP(remoteAddr) != nil {
+					this.remoteAddr = remoteAddr
+					return remoteAddr
+				}
+			}
+		} else { // single value
+			var remoteAddr = this.Format(this.web.RemoteAddr.Value)
+			if len(remoteAddr) > 0 && net.ParseIP(remoteAddr) != nil {
+				this.remoteAddr = remoteAddr
+				return remoteAddr
+			}
+		}
+
+		// 如果是从Header中读取，则直接返回原始IP
+		if this.web.RemoteAddr.Type == serverconfigs.HTTPRemoteAddrTypeRequestHeader {
+			var remoteAddr = this.RawReq.RemoteAddr
+			host, _, err := net.SplitHostPort(remoteAddr)
+			if err == nil {
+				this.remoteAddr = host
+				return host
+			} else {
+				return remoteAddr
+			}
 		}
 	}
 
@@ -1222,7 +1262,7 @@ func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
 	var forwardedFor = this.RawReq.Header.Get("X-Forwarded-For")
 	if len(forwardedFor) > 0 {
 		commaIndex := strings.Index(forwardedFor, ",")
-		if commaIndex > 0 {
+		if commaIndex > 0 && !lists.ContainsString(result, forwardedFor[:commaIndex]) {
 			result = append(result, forwardedFor[:commaIndex])
 		}
 	}
@@ -1230,7 +1270,7 @@ func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
 	// Real-IP
 	{
 		realIP, ok := this.RawReq.Header["X-Real-IP"]
-		if ok && len(realIP) > 0 {
+		if ok && len(realIP) > 0 && !lists.ContainsString(result, realIP[0]) {
 			result = append(result, realIP[0])
 		}
 	}
@@ -1238,7 +1278,7 @@ func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
 	// Real-Ip
 	{
 		realIP, ok := this.RawReq.Header["X-Real-Ip"]
-		if ok && len(realIP) > 0 {
+		if ok && len(realIP) > 0 && !lists.ContainsString(result, realIP[0]) {
 			result = append(result, realIP[0])
 		}
 	}
@@ -1248,7 +1288,9 @@ func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
 		var remoteAddr = this.RawReq.RemoteAddr
 		host, _, err := net.SplitHostPort(remoteAddr)
 		if err == nil {
-			result = append(result, host)
+			if !lists.ContainsString(result, host) {
+				result = append(result, host)
+			}
 		} else {
 			result = append(result, remoteAddr)
 		}
@@ -1565,10 +1607,10 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
 		this.RawReq.Header.Set("Connection", "keep-alive")
 	}
 
-	remoteAddr := this.RawReq.RemoteAddr
-	host, _, err := net.SplitHostPort(remoteAddr)
+	var rawRemoteAddr = this.RawReq.RemoteAddr
+	host, _, err := net.SplitHostPort(rawRemoteAddr)
 	if err == nil {
-		remoteAddr = host
+		rawRemoteAddr = host
 	}
 
 	// x-real-ip
@@ -1576,24 +1618,24 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
 		_, ok1 := header["X-Real-IP"]
 		_, ok2 := header["X-Real-Ip"]
 		if !ok1 && !ok2 {
-			header["X-Real-IP"] = []string{remoteAddr}
+			header["X-Real-IP"] = []string{this.requestRemoteAddr(true)}
 		}
 	}
 
 	// X-Forwarded-For
 	if this.reverseProxy != nil && this.reverseProxy.ShouldAddXForwardedForHeader() {
 		forwardedFor, ok := header["X-Forwarded-For"]
-		if ok {
+		if ok && len(forwardedFor) > 0 { // already exists
 			_, hasForwardHeader := this.RawReq.Header["X-Forwarded-For"]
 			if hasForwardHeader {
-				header["X-Forwarded-For"] = []string{strings.Join(forwardedFor, ", ") + ", " + remoteAddr}
+				header["X-Forwarded-For"] = []string{strings.Join(forwardedFor, ", ") + ", " + rawRemoteAddr}
 			}
 		} else {
 			var clientRemoteAddr = this.requestRemoteAddr(true)
-			if len(clientRemoteAddr) > 0 && clientRemoteAddr != remoteAddr {
-				header["X-Forwarded-For"] = []string{clientRemoteAddr + ", " + remoteAddr}
+			if len(clientRemoteAddr) > 0 && clientRemoteAddr != rawRemoteAddr {
+				header["X-Forwarded-For"] = []string{clientRemoteAddr + ", " + rawRemoteAddr}
 			} else {
-				header["X-Forwarded-For"] = []string{remoteAddr}
+				header["X-Forwarded-For"] = []string{rawRemoteAddr}
 			}
 		}
 	}

internal/nodes/http_request_cache.go

@@ -274,7 +274,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		}
 
 		if err != nil {
-			if err == caches.ErrNotFound {
+			if errors.Is(err, caches.ErrNotFound) {
+				// 移除请求中的 If-None-Match 和 If-Modified-Since，防止源站返回304而无法缓存
+				if this.reverseProxy != nil {
+					this.RawReq.Header.Del("If-None-Match")
+					this.RawReq.Header.Del("If-Modified-Since")
+				}
+
 				// cache相关变量
 				this.varMapping["cache.status"] = "MISS"
 
@@ -365,24 +371,24 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	}
 
 	// ETag
-	// 这里强制设置ETag，如果先前源站设置了ETag，将会被覆盖，避免因为源站的ETag导致源站返回304 Not Modified
 	var respHeader = this.writer.Header()
-	var eTag = ""
+	var eTag = respHeader.Get("ETag")
 	var lastModifiedAt = reader.LastModified()
-	if lastModifiedAt > 0 {
-		if len(tags) > 0 {
-			eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
-		} else {
-			eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
-		}
-		respHeader.Del("Etag")
-		if !isPartialCache {
-			respHeader["ETag"] = []string{eTag}
+	if len(eTag) == 0 {
+		if lastModifiedAt > 0 {
+			if len(tags) > 0 {
+				eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
+			} else {
+				eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
+			}
+			respHeader.Del("Etag")
+			if !isPartialCache {
+				respHeader["ETag"] = []string{eTag}
+			}
 		}
 	}
 
 	// 支持 Last-Modified
-	// 这里强制设置Last-Modified，如果先前源站设置了Last-Modified，将会被覆盖，避免因为源站的Last-Modified导致源站返回304 Not Modified
 	var modifiedTime = ""
 	if lastModifiedAt > 0 {
 		modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
@@ -490,7 +496,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 			if err != nil {
 				this.varMapping["cache.status"] = "MISS"
 
-				if err == caches.ErrInvalidRange {
+				if errors.Is(err, caches.ErrInvalidRange) {
 					this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
 					this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 					return true

internal/nodes/http_request_error.go

@@ -9,16 +9,20 @@ import (
 )
 
 const httpStatusPageTemplate = `<!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>${status} ${statusMessage}</title>
 	<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+	<style>
+	address { line-height: 1.8; }
+	</style>
 </head>
 <body>
 
 <h1>${status} ${statusMessage}</h1>
 <p>${message}</p>
 
+<address>Connection: ${remoteAddr} (Client) -&gt; ${serverAddr} (Server)</address>
 <address>Request ID: ${requestId}.</address>
 
 </body>
@@ -39,8 +43,6 @@ func (this *HTTPRequest) writeCode(statusCode int, enMessage string, zhMessage s
 			return types.String(statusCode)
 		case "statusMessage":
 			return http.StatusText(statusCode)
-		case "requestId":
-			return this.requestId
 		case "message":
 			var acceptLanguages = this.RawReq.Header.Get("Accept-Language")
 			if len(acceptLanguages) > 0 {
@@ -54,7 +56,7 @@ func (this *HTTPRequest) writeCode(statusCode int, enMessage string, zhMessage s
 			}
 			return enMessage
 		}
-		return "${" + varName + "}"
+		return this.Format("${" + varName + "}")
 	})
 
 	this.ProcessResponseHeaders(this.writer.Header(), statusCode)
@@ -107,7 +109,7 @@ func (this *HTTPRequest) write50x(err error, statusCode int, enMessage string, z
 			}
 			return "The site is unavailable now, cause: " + enMessage + "."
 		}
-		return "${" + varName + "}"
+		return this.Format("${" + varName + "}")
 	})
 
 	this.ProcessResponseHeaders(this.writer.Header(), statusCode)

internal/nodes/http_request_host_redirect.go

@@ -25,6 +25,13 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 			continue
 		}
 
+		if len(u.ExceptDomains) > 0 && configutils.MatchDomains(u.ExceptDomains, this.ReqHost) {
+			continue
+		}
+		if len(u.OnlyDomains) > 0 && !configutils.MatchDomains(u.OnlyDomains, this.ReqHost) {
+			continue
+		}
+
 		var status = u.Status
 		if status <= 0 {
 			if searchEngineRegex.MatchString(this.RawReq.UserAgent()) {
@@ -55,7 +62,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 					}
 
 					this.ProcessResponseHeaders(this.writer.Header(), status)
-					http.Redirect(this.RawWriter, this.RawReq, afterURL, status)
+					httpRedirect(this.writer, this.RawReq, afterURL, status)
 					return true
 				}
 			} else if u.MatchRegexp { // 正则匹配
@@ -97,7 +104,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 				}
 
 				this.ProcessResponseHeaders(this.writer.Header(), status)
-				http.Redirect(this.RawWriter, this.RawReq, afterURL, status)
+				httpRedirect(this.writer, this.RawReq, afterURL, status)
 				return true
 			} else { // 精准匹配
 				if fullURL == u.RealBeforeURL() {
@@ -120,7 +127,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 					}
 
 					this.ProcessResponseHeaders(this.writer.Header(), status)
-					http.Redirect(this.RawWriter, this.RawReq, afterURL, status)
+					httpRedirect(this.writer, this.RawReq, afterURL, status)
 					return true
 				}
 			}
@@ -139,11 +146,6 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 				}
 			}
 
-			// 如果跳转前后域名一致，则终止
-			if u.DomainAfter == reqHost {
-				return false
-			}
-
 			var scheme = u.DomainAfterScheme
 			if len(scheme) == 0 {
 				scheme = this.requestScheme()
@@ -155,6 +157,11 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 					return false
 				}
 
+				// 如果跳转前后域名一致，则终止
+				if u.DomainAfter == reqHost {
+					return false
+				}
+
 				this.ProcessResponseHeaders(this.writer.Header(), status)
 
 				// 参数
@@ -163,7 +170,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 					afterURL += this.uri[qIndex:]
 				}
 
-				http.Redirect(this.RawWriter, this.RawReq, afterURL, status)
+				httpRedirect(this.writer, this.RawReq, afterURL, status)
 				return true
 			}
 		} else if u.Type == serverconfigs.HTTPHostRedirectTypePort {
@@ -212,7 +219,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 				}
 
 				this.ProcessResponseHeaders(this.writer.Header(), status)
-				http.Redirect(this.RawWriter, this.RawReq, afterURL, status)
+				httpRedirect(this.writer, this.RawReq, afterURL, status)
 				return true
 			}
 		}

internal/nodes/http_request_ln.go

@@ -20,6 +20,6 @@ func (this *HTTPRequest) checkLnRequest() bool {
 	return false
 }
 
-func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
+func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64, urlHash uint64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
 	return nil, 0, false
 }

internal/nodes/http_request_mismatch.go

@@ -7,9 +7,9 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/waf"
 	"github.com/iwind/TeaGo/types"
-	"net"
 	"net/http"
 	"time"
 )
@@ -48,7 +48,8 @@ func (this *HTTPRequest) doMismatch() {
 		}
 
 		// 是否正在访问IP
-		if globalServerConfig.HTTPAll.NodeIPShowPage && net.ParseIP(this.ReqHost) != nil {
+		if globalServerConfig.HTTPAll.NodeIPShowPage && utils.IsWildIP(this.ReqHost) {
+			this.writer.statusCode = statusCode
 			var contentHTML = this.Format(globalServerConfig.HTTPAll.NodeIPPageHTML)
 			this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 			this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
@@ -62,7 +63,7 @@ func (this *HTTPRequest) doMismatch() {
 		// 要考虑到服务在切换集群时，域名未生效状态时，用户访问的仍然是老集群中的节点，就会产生找不到域名的情况
 		if len(remoteIP) > 0 {
 			const maxAttempts = 100
-			if ttlcache.SharedCache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
+			if ttlcache.SharedInt64Cache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
 				// 在加入之前再次检查黑名单
 				if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
 					waf.SharedIPBlackList.Add(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+3600)
@@ -71,8 +72,9 @@ func (this *HTTPRequest) doMismatch() {
 		}
 
 		// 处理当前连接
-		if mismatchAction != nil && mismatchAction.Code == "page" {
+		if mismatchAction != nil && mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
 			if mismatchAction.Options != nil {
+				this.writer.statusCode = statusCode
 				var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
 				this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 				this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))

internal/nodes/http_request_page.go

@@ -2,7 +2,6 @@ package nodes
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/Tea"
@@ -46,9 +45,13 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
 }
 
 func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, status int) (shouldStop bool) {
+	var url = this.URL()
 	for _, page := range pages {
+		if !page.MatchURL(url) {
+			continue
+		}
 		if page.Match(status) {
-			if len(page.BodyType) == 0 || page.BodyType == shared.BodyTypeURL {
+			if len(page.BodyType) == 0 || page.BodyType == serverconfigs.HTTPPageBodyTypeURL {
 				if urlSchemeRegexp.MatchString(page.URL) {
 					var newStatus = page.NewStatus
 					if newStatus <= 0 {
@@ -115,7 +118,7 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
 				}
 
 				return true
-			} else if page.BodyType == shared.BodyTypeHTML {
+			} else if page.BodyType == serverconfigs.HTTPPageBodyTypeHTML {
 				// 这里需要实现设置Status，因为在Format()中可以获取${status}等变量
 				if page.NewStatus > 0 {
 					this.writer.statusCode = page.NewStatus
@@ -147,6 +150,18 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
 					this.writer.SetOk()
 				}
 				return true
+			} else if page.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
+				var newURL = page.URL
+				if len(newURL) == 0 {
+					newURL = "/"
+				}
+				if page.NewStatus > 0 && httpStatusIsRedirect(page.NewStatus) {
+					httpRedirect(this.writer, this.RawReq, newURL, page.NewStatus)
+				} else {
+					httpRedirect(this.writer, this.RawReq, newURL, http.StatusTemporaryRedirect)
+				}
+				this.writer.SetOk()
+				return true
 			}
 		}
 	}

internal/nodes/http_request_redirect_https.go

@@ -43,7 +43,7 @@ func (this *HTTPRequest) doRedirectToHTTPS(redirectToHTTPSConfig *serverconfigs.
 
 	var newURL = "https://" + host + this.RawReq.RequestURI
 	this.ProcessResponseHeaders(this.writer.Header(), statusCode)
-	http.Redirect(this.writer, this.RawReq, newURL, statusCode)
+	httpRedirect(this.writer, this.RawReq, newURL, statusCode)
 
 	return true
 }

internal/nodes/http_request_reverse_proxy.go

@@ -7,6 +7,8 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"io"
 	"net/http"
@@ -25,9 +27,10 @@ func (this *HTTPRequest) doReverseProxy() {
 
 	var failedOriginIds []int64
 	var failedLnNodeIds []int64
+	var failStatusCode int
 
 	for i := 0; i < retries; i++ {
-		originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1)
+		originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1, &failStatusCode)
 		if !shouldRetry {
 			break
 		}
@@ -41,7 +44,7 @@ func (this *HTTPRequest) doReverseProxy() {
 }
 
 // 请求源站
-func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool) (originId int64, lnNodeId int64, shouldRetry bool) {
+func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool, failStatusCode *int) (originId int64, lnNodeId int64, shouldRetry bool) {
 	// 对URL的处理
 	var stripPrefix = this.reverseProxy.StripPrefix
 	var requestURI = this.reverseProxy.RequestURI
@@ -65,7 +68,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 	// 二级节点
 	var hasMultipleLnNodes = false
 	if this.cacheRef != nil || (this.nodeConfig != nil && this.nodeConfig.GlobalServerConfig != nil && this.nodeConfig.GlobalServerConfig.HTTPAll.ForceLnRequest) {
-		origin, lnNodeId, hasMultipleLnNodes = this.getLnOrigin(failedLnNodeIds)
+		origin, lnNodeId, hasMultipleLnNodes = this.getLnOrigin(failedLnNodeIds, fnv.HashString(this.URL()))
 		if origin != nil {
 			// 强制变更原来访问的域名
 			requestHost = this.ReqHost
@@ -89,6 +92,10 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 		}
 		if origin == nil {
 			origin = this.reverseProxy.NextOrigin(requestCall)
+			if origin != nil && origin.Id > 0 && (*failStatusCode >= 403 && *failStatusCode <= 404) && lists.ContainsInt64(failedOriginIds, origin.Id) {
+				this.writeCode(*failStatusCode, "", "")
+				return
+			}
 		}
 		requestCall.CallResponseCallbacks(this.writer)
 		if origin == nil {
@@ -306,7 +313,8 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 
 	if requestErr != nil {
 		// 客户端取消请求，则不提示
-		httpErr, ok := requestErr.(*url.Error)
+		var httpErr *url.Error
+		ok := errors.As(requestErr, &httpErr)
 		if !ok {
 			if isHTTPOrigin {
 				SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
@@ -320,7 +328,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 				this.write50x(requestErr, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
 			}
 			remotelogs.WarnServer("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+": Request origin server failed: "+requestErr.Error())
-		} else if httpErr.Err != context.Canceled {
+		} else if !errors.Is(httpErr, context.Canceled) {
 			if isHTTPOrigin {
 				SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
 					this.reverseProxy.ResetScheduling()
@@ -332,7 +340,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 				shouldRetry = true
 				this.uri = oldURI // 恢复备份
 
-				if httpErr.Err != io.EOF {
+				if httpErr.Err != io.EOF && !errors.Is(httpErr.Err, http.ErrBodyReadAfterClose) {
 					remotelogs.WarnServer("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+requestErr.Error())
 				}
 
@@ -346,14 +354,14 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 			} else {
 				this.write50x(requestErr, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
 			}
-			if httpErr.Err != io.EOF {
+			if httpErr.Err != io.EOF && !errors.Is(httpErr.Err, http.ErrBodyReadAfterClose) {
 				remotelogs.WarnServer("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+requestErr.Error())
 			}
 		} else {
 			// 是否为客户端方面的错误
 			var isClientError = false
 			if ok {
-				if httpErr.Err == context.Canceled {
+				if errors.Is(httpErr, context.Canceled) {
 					// 如果是服务器端主动关闭，则无需提示
 					if this.isConnClosed() {
 						this.disableLog = true
@@ -373,6 +381,35 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 		return
 	}
 
+	// 40x && 50x
+	*failStatusCode = resp.StatusCode
+	if resp != nil &&
+		((resp.StatusCode >= 500 && resp.StatusCode < 510 && this.reverseProxy.Retry50X) ||
+			(resp.StatusCode >= 403 && resp.StatusCode <= 404 && this.reverseProxy.Retry40X)) &&
+		(originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) &&
+		!isLastRetry {
+		if resp.Body != nil {
+			_ = resp.Body.Close()
+		}
+
+		shouldRetry = true
+		return
+	}
+
+	// 尝试从缓存中恢复
+	if resp != nil &&
+		resp.StatusCode >= 500 && // support 50X only
+		resp.StatusCode < 510 &&
+		this.cacheCanTryStale &&
+		this.web.Cache.Stale != nil &&
+		this.web.Cache.Stale.IsOn &&
+		(len(this.web.Cache.Stale.Status) == 0 || lists.ContainsInt(this.web.Cache.Stale.Status, resp.StatusCode)) {
+		var ok = this.doCacheRead(true)
+		if ok {
+			return
+		}
+	}
+
 	// 记录相关数据
 	this.originStatus = int32(resp.StatusCode)
 

internal/nodes/http_request_rewrite.go

@@ -31,10 +31,10 @@ func (this *HTTPRequest) doRewrite() (shouldShop bool) {
 	if this.rewriteRule.Mode == serverconfigs.HTTPRewriteModeRedirect {
 		if this.rewriteRule.RedirectStatus > 0 {
 			this.ProcessResponseHeaders(this.writer.Header(), this.rewriteRule.RedirectStatus)
-			http.Redirect(this.writer, this.RawReq, this.rewriteReplace, this.rewriteRule.RedirectStatus)
+			httpRedirect(this.writer, this.RawReq, this.rewriteReplace, this.rewriteRule.RedirectStatus)
 		} else {
 			this.ProcessResponseHeaders(this.writer.Header(), http.StatusTemporaryRedirect)
-			http.Redirect(this.writer, this.RawReq, this.rewriteReplace, http.StatusTemporaryRedirect)
+			httpRedirect(this.writer, this.RawReq, this.rewriteReplace, http.StatusTemporaryRedirect)
 		}
 		return true
 	}

internal/nodes/http_request_shutdown.go

@@ -1,7 +1,7 @@
 package nodes
 
 import (
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/Tea"
@@ -18,7 +18,7 @@ func (this *HTTPRequest) doShutdown() {
 		return
 	}
 
-	if len(shutdown.BodyType) == 0 || shutdown.BodyType == shared.BodyTypeURL {
+	if len(shutdown.BodyType) == 0 || shutdown.BodyType == serverconfigs.HTTPPageBodyTypeURL {
 		// URL
 		if urlSchemeRegexp.MatchString(shutdown.URL) {
 			this.doURL(http.MethodGet, shutdown.URL, "", shutdown.Status, true)
@@ -80,7 +80,7 @@ func (this *HTTPRequest) doShutdown() {
 		} else {
 			this.writer.SetOk()
 		}
-	} else if shutdown.BodyType == shared.BodyTypeHTML {
+	} else if shutdown.BodyType == serverconfigs.HTTPPageBodyTypeHTML {
 		// 自定义响应Headers
 		if shutdown.Status > 0 {
 			this.ProcessResponseHeaders(this.writer.Header(), shutdown.Status)
@@ -98,5 +98,17 @@ func (this *HTTPRequest) doShutdown() {
 		} else {
 			this.writer.SetOk()
 		}
+	} else if shutdown.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
+		var newURL = shutdown.URL
+		if len(newURL) == 0 {
+			newURL = "/"
+		}
+
+		if shutdown.Status > 0 && httpStatusIsRedirect(shutdown.Status) {
+			httpRedirect(this.writer, this.RawReq, newURL, shutdown.Status)
+		} else {
+			httpRedirect(this.writer, this.RawReq, newURL, http.StatusTemporaryRedirect)
+		}
+		this.writer.SetOk()
 	}
 }

internal/nodes/http_request_traffic_limit.go

@@ -8,15 +8,17 @@ import (
 
 // 流量限制
 func (this *HTTPRequest) doTrafficLimit() {
-	var config = this.ReqServer.TrafficLimit
-
-	this.tags = append(this.tags, "bandwidth")
+	this.tags = append(this.tags, "trafficLimit")
 
 	var statusCode = 509
+	this.writer.statusCode = statusCode
 	this.ProcessResponseHeaders(this.writer.Header(), statusCode)
 
+	this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 	this.writer.WriteHeader(statusCode)
-	if len(config.NoticePageBody) != 0 {
+
+	var config = this.ReqServer.TrafficLimit
+	if config != nil && len(config.NoticePageBody) != 0 {
 		_, _ = this.writer.WriteString(this.Format(config.NoticePageBody))
 	} else {
 		_, _ = this.writer.WriteString(this.Format(serverconfigs.DefaultTrafficLimitNoticePageBody))

internal/nodes/http_request_waf.go

@@ -96,6 +96,8 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 		return
 	}
 
+	var isDefendMode = firewallPolicy.Mode == firewallconfigs.FirewallModeDefend
+
 	// 检查IP白名单
 	var remoteAddrs []string
 	if len(this.remoteAddr) > 0 {
@@ -122,7 +124,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 	}
 
 	// 检查IP黑名单
-	if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
+	if isDefendMode {
 		for _, ref := range inbound.AllDenyListRefs() {
 			if ref.IsOn && ref.ListId > 0 {
 				list := iplibrary.SharedIPListManager.FindList(ref.ListId)
@@ -161,19 +163,20 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 	}
 
 	// 检查地区封禁
-	if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
-		if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
-			var regionConfig = firewallPolicy.Inbound.Region
-			if regionConfig.IsNotEmpty() {
-				for _, remoteAddr := range remoteAddrs {
-					var result = iplib.LookupIP(remoteAddr)
-					if result != nil && result.IsOk() {
-						var currentURL = this.URL()
-						if regionConfig.MatchCountryURL(currentURL) {
-							// 检查国家/地区级别封禁
-							if !regionConfig.IsAllowedCountry(result.CountryId(), result.ProvinceId()) {
-								this.firewallPolicyId = firewallPolicy.Id
 
+	if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
+		var regionConfig = firewallPolicy.Inbound.Region
+		if regionConfig.IsNotEmpty() {
+			for _, remoteAddr := range remoteAddrs {
+				var result = iplib.LookupIP(remoteAddr)
+				if result != nil && result.IsOk() {
+					var currentURL = this.URL()
+					if regionConfig.MatchCountryURL(currentURL) {
+						// 检查国家/地区级别封禁
+						if !regionConfig.IsAllowedCountry(result.CountryId(), result.ProvinceId()) {
+							this.firewallPolicyId = firewallPolicy.Id
+
+							if isDefendMode {
 								var promptHTML string
 								if len(regionConfig.CountryHTML) > 0 {
 									promptHTML = regionConfig.CountryHTML
@@ -193,23 +196,27 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 
 								// 延时返回，避免攻击
 								time.Sleep(1 * time.Second)
+							}
 
-								// 停止日志
-								if !logDenying {
-									this.disableLog = true
-								} else {
-									this.tags = append(this.tags, "denyCountry")
-								}
+							// 停止日志
+							if !logDenying {
+								this.disableLog = true
+							} else {
+								this.tags = append(this.tags, "denyCountry")
+							}
 
+							if isDefendMode {
 								return true, false
 							}
 						}
+					}
 
-						if regionConfig.MatchProvinceURL(currentURL) {
-							// 检查省份封禁
-							if !regionConfig.IsAllowedProvince(result.CountryId(), result.ProvinceId()) {
-								this.firewallPolicyId = firewallPolicy.Id
+					if regionConfig.MatchProvinceURL(currentURL) {
+						// 检查省份封禁
+						if !regionConfig.IsAllowedProvince(result.CountryId(), result.ProvinceId()) {
+							this.firewallPolicyId = firewallPolicy.Id
 
+							if isDefendMode {
 								var promptHTML string
 								if len(regionConfig.ProvinceHTML) > 0 {
 									promptHTML = regionConfig.ProvinceHTML
@@ -229,14 +236,16 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 
 								// 延时返回，避免攻击
 								time.Sleep(1 * time.Second)
+							}
 
-								// 停止日志
-								if !logDenying {
-									this.disableLog = true
-								} else {
-									this.tags = append(this.tags, "denyProvince")
-								}
+							// 停止日志
+							if !logDenying {
+								this.disableLog = true
+							} else {
+								this.tags = append(this.tags, "denyProvince")
+							}
 
+							if isDefendMode {
 								return true, false
 							}
 						}
@@ -257,7 +266,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 		return
 	}
 
-	goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer)
+	goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer, this.web.FirewallRef.DefaultCaptchaType)
 	if forceLog && logRequestBody && hasRequestBody && ruleSet != nil && ruleSet.HasAttackActions() {
 		this.wafHasRequestBody = true
 	}
@@ -307,7 +316,7 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
 	}
 
 	if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
-		blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
+		blocked = this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
 		if blocked {
 			return true
 		}
@@ -315,7 +324,7 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
 
 	// 公用的防火墙设置
 	if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
-		blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
+		blocked = this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
 		if blocked {
 			return true
 		}
@@ -469,3 +478,10 @@ func (this *HTTPRequest) WAFMaxRequestSize() int64 {
 func (this *HTTPRequest) DisableAccessLog() {
 	this.disableLog = true
 }
+
+// DisableStat 停用统计
+func (this *HTTPRequest) DisableStat() {
+	if this.web != nil {
+		this.web.StatRef = nil
+	}
+}

internal/nodes/http_writer.go

@@ -16,6 +16,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/readers"
+	setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
 	_ "github.com/biessek/golang-ico"
 	"github.com/iwind/TeaGo/lists"
@@ -39,6 +40,7 @@ import (
 
 var webpMaxBufferSize int64 = 1_000_000_000
 var webpTotalBufferSize int64 = 0
+var webpIgnoreURLSet = setutils.NewFixedSet(131072)
 
 func init() {
 	if !teaconst.IsMain {
@@ -47,7 +49,7 @@ func init() {
 
 	var systemMemory = utils.SystemMemoryGB() / 8
 	if systemMemory > 0 {
-		webpMaxBufferSize = int64(systemMemory) * 1024 * 1024 * 1024
+		webpMaxBufferSize = int64(systemMemory) << 30
 	}
 }
 
@@ -547,6 +549,11 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
 		this.req.web.WebP.IsOn &&
 		this.req.web.WebP.MatchResponse(contentType, size, filepath.Ext(this.req.Path()), this.req.Format) &&
 		this.req.web.WebP.MatchAccept(this.req.requestHeader("Accept")) {
+		// 检查是否已经因为尺寸过大而忽略
+		if webpIgnoreURLSet.Has(this.req.URL()) {
+			return
+		}
+
 		// 如果已经是WebP不再重复处理
 		// TODO 考虑是否需要很严格的匹配
 		if strings.Contains(contentType, "image/webp") {
@@ -787,12 +794,10 @@ func (this *HTTPWriter) AddHeaders(header http.Header) {
 			continue
 		}
 		switch key {
-		case "ETag":
+		case "Accept-CH", "ETag", "Content-MD5", "IM", "P3P", "WWW-Authenticate", "X-Request-ID":
 			newHeaders[key] = value
 		default:
-			for _, v := range value {
-				newHeaders.Add(key, v)
-			}
+			newHeaders[http.CanonicalHeaderKey(key)] = value
 		}
 	}
 }
@@ -908,7 +913,7 @@ func (this *HTTPWriter) SendResp(resp *http.Response) (int64, error) {
 
 // Redirect 跳转
 func (this *HTTPWriter) Redirect(status int, url string) {
-	http.Redirect(this.rawWriter, this.req.RawReq, url, status)
+	httpRedirect(this, this.req.RawReq, url, status)
 	this.isFinished = true
 }
 
@@ -984,7 +989,7 @@ func (this *HTTPWriter) DelayRead() bool {
 
 // 计算stale时长
 func (this *HTTPWriter) calculateStaleLife() int {
-	var staleLife = 600 // TODO 可以在缓存策略里设置此时间
+	var staleLife = caches.DefaultStaleCacheSeconds
 	var staleConfig = this.req.web.Cache.Stale
 	if staleConfig != nil && staleConfig.IsOn {
 		// 从Header中读取stale-if-error
@@ -1074,12 +1079,24 @@ func (this *HTTPWriter) finishWebP() {
 		var err error
 		if isGif {
 			gifImage, err = gif.DecodeAll(reader)
+			if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
+				webpIgnoreURLSet.Push(this.req.URL())
+				return
+			}
 		} else {
 			imageData, _, err = image.Decode(reader)
+			if imageData != nil {
+				var bound = imageData.Bounds()
+				if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
+					webpIgnoreURLSet.Push(this.req.URL())
+					return
+				}
+			}
 		}
 
 		if err != nil {
 			// 发生了错误终止处理
+			webpIgnoreURLSet.Push(this.req.URL())
 			return
 		}
 
@@ -1267,7 +1284,7 @@ func (this *HTTPWriter) calculateHeaderLength() (result int) {
 
 func (this *HTTPWriter) shouldIgnoreHeader(name string) bool {
 	switch name {
-	case "Set-Cookie", "Strict-Transport-Security", "Alt-Svc", "Upgrade":
+	case "Set-Cookie", "Strict-Transport-Security", "Alt-Svc", "Upgrade", "X-Cache":
 		return true
 	default:
 		return (this.isPartial && name == "Content-Range")

internal/nodes/listener_base.go

@@ -7,6 +7,7 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/types"
 	"net"
 )
@@ -45,7 +46,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
 				}
 			}
 
-			tlsPolicy, _, err := this.matchSSL(this.helloServerName(clientInfo))
+			tlsPolicy, _, err := this.matchSSL(this.helloServerNames(clientInfo))
 			if err != nil {
 				return nil, err
 			}
@@ -68,7 +69,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
 				}
 			}
 
-			tlsPolicy, cert, err := this.matchSSL(this.helloServerName(clientInfo))
+			tlsPolicy, cert, err := this.matchSSL(this.helloServerNames(clientInfo))
 			if err != nil {
 				return nil, err
 			}
@@ -84,7 +85,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
 }
 
 // 根据域名匹配证书
-func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
+func (this *BaseListener) matchSSL(domains []string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
 	var group = this.Group
 
 	if group == nil {
@@ -98,7 +99,7 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 
 	// 如果域名为空，则取第一个
 	// 通常域名为空是因为是直接通过IP访问的
-	if len(domain) == 0 {
+	if len(domains) == 0 {
 		if group.IsHTTPS() && globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
 			return nil, nil, errors.New("no tls server name matched")
 		}
@@ -115,9 +116,25 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 		}
 		return nil, nil, errors.New("no tls server name found")
 	}
+	var firstDomain = domains[0]
 
 	// 通过网站域名配置匹配
-	server, _ := this.findNamedServer(domain)
+	var server *serverconfigs.ServerConfig
+	var matchedDomain string
+	for _, domain := range domains {
+		server, _ = this.findNamedServer(domain, true)
+		if server != nil {
+			matchedDomain = domain
+			break
+		}
+	}
+	if server == nil {
+		server, _ = this.findNamedServer(firstDomain, false)
+		if server != nil {
+			matchedDomain = firstDomain
+		}
+	}
+
 	if server == nil {
 		// 找不到或者此时的服务没有配置证书，需要搜索所有的Server，通过SSL证书内容中的DNSName匹配
 		// 此功能仅为了兼容以往版本（v1.0.4），不应该作为常态启用
@@ -126,14 +143,14 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 				if searchingServer.SSLPolicy() == nil || !searchingServer.SSLPolicy().IsOn {
 					continue
 				}
-				cert, ok := searchingServer.SSLPolicy().MatchDomain(domain)
+				cert, ok := searchingServer.SSLPolicy().MatchDomain(firstDomain)
 				if ok {
 					return searchingServer.SSLPolicy(), cert, nil
 				}
 			}
 		}
 
-		return nil, nil, errors.New("no server found for '" + domain + "'")
+		return nil, nil, errors.New("no server found for '" + firstDomain + "'")
 	}
 	if server.SSLPolicy() == nil || !server.SSLPolicy().IsOn {
 		// 找不到或者此时的服务没有配置证书，需要搜索所有的Server，通过SSL证书内容中的DNSName匹配
@@ -143,32 +160,32 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 				if searchingServer.SSLPolicy() == nil || !searchingServer.SSLPolicy().IsOn {
 					continue
 				}
-				cert, ok := searchingServer.SSLPolicy().MatchDomain(domain)
+				cert, ok := searchingServer.SSLPolicy().MatchDomain(matchedDomain)
 				if ok {
 					return searchingServer.SSLPolicy(), cert, nil
 				}
 			}
 		}
 
-		return nil, nil, errors.New("no cert found for '" + domain + "'")
+		return nil, nil, errors.New("no cert found for '" + matchedDomain + "'")
 	}
 
 	// 证书是否匹配
 	var sslConfig = server.SSLPolicy()
-	cert, ok := sslConfig.MatchDomain(domain)
+	cert, ok := sslConfig.MatchDomain(matchedDomain)
 	if ok {
 		return sslConfig, cert, nil
 	}
 
 	if len(sslConfig.Certs) == 0 {
-		remotelogs.ServerError(server.Id, "BASE_LISTENER", "no ssl certs found for '"+domain+"', server id: "+types.String(server.Id), "", nil)
+		remotelogs.ServerError(server.Id, "BASE_LISTENER", "no ssl certs found for '"+matchedDomain+"', server id: "+types.String(server.Id), "", nil)
 	}
 
 	return sslConfig, sslConfig.FirstCert(), nil
 }
 
 // 根据域名来查找匹配的域名
-func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconfigs.ServerConfig, serverName string) {
+func (this *BaseListener) findNamedServer(name string, exactly bool) (serverConfig *serverconfigs.ServerConfig, serverName string) {
 	serverConfig, serverName = this.findNamedServerMatched(name)
 	if serverConfig != nil {
 		return
@@ -179,10 +196,10 @@ func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconf
 
 	if globalServerConfig != nil &&
 		len(globalServerConfig.HTTPAll.DefaultDomain) > 0 &&
-		(!matchDomainStrictly || configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) || (globalServerConfig.HTTPAll.AllowNodeIP && net.ParseIP(name) != nil)) {
+		(!matchDomainStrictly || configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) || (globalServerConfig.HTTPAll.AllowNodeIP && utils.IsWildIP(name))) {
 		if globalServerConfig.HTTPAll.AllowNodeIP &&
 			globalServerConfig.HTTPAll.NodeIPShowPage &&
-			net.ParseIP(name) != nil {
+			utils.IsWildIP(name) {
 			return
 		} else {
 			var defaultDomain = globalServerConfig.HTTPAll.DefaultDomain
@@ -193,18 +210,22 @@ func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconf
 		}
 	}
 
-	if matchDomainStrictly && !configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) && (!globalServerConfig.HTTPAll.AllowNodeIP || net.ParseIP(name) == nil) {
+	if matchDomainStrictly && !configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) && (!globalServerConfig.HTTPAll.AllowNodeIP || (!utils.IsWildIP(name) || globalServerConfig.HTTPAll.NodeIPShowPage)) {
 		return
 	}
 
-	// 如果没有找到，则匹配到第一个
-	var group = this.Group
-	var currentServers = group.Servers()
-	var countServers = len(currentServers)
-	if countServers == 0 {
-		return nil, ""
+	if !exactly {
+		// 如果没有找到，则匹配到第一个
+		var group = this.Group
+		var currentServers = group.Servers()
+		var countServers = len(currentServers)
+		if countServers == 0 {
+			return nil, ""
+		}
+		return currentServers[0], name
 	}
-	return currentServers[0], name
+
+	return
 }
 
 // 严格查找域名
@@ -233,16 +254,23 @@ func (this *BaseListener) findNamedServerMatched(name string) (serverConfig *ser
 }
 
 // 从Hello信息中获取服务名称
-func (this *BaseListener) helloServerName(clientInfo *tls.ClientHelloInfo) string {
-	var serverName = clientInfo.ServerName
-	if len(serverName) == 0 && clientInfo.Conn != nil {
+func (this *BaseListener) helloServerNames(clientInfo *tls.ClientHelloInfo) (serverNames []string) {
+	if len(clientInfo.ServerName) != 0 {
+		serverNames = append(serverNames, clientInfo.ServerName)
+		return
+	}
+
+	if clientInfo.Conn != nil {
 		var localAddr = clientInfo.Conn.LocalAddr()
 		if localAddr != nil {
 			tcpAddr, ok := localAddr.(*net.TCPAddr)
 			if ok {
-				serverName = tcpAddr.IP.String()
+				serverNames = append(serverNames, tcpAddr.IP.String())
 			}
 		}
 	}
-	return serverName
+
+	serverNames = append(serverNames, sharedNodeConfig.IPAddresses...)
+
+	return
 }

internal/nodes/listener_http.go

@@ -107,15 +107,23 @@ func (this *HTTPListener) Reload(group *serverconfigs.ServerAddressGroup) {
 
 // ServerHTTP 处理HTTP请求
 func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.Request) {
+	if len(rawReq.Host) > 253 {
+		http.Error(rawWriter, "Host too long.", http.StatusBadRequest)
+		time.Sleep(1 * time.Second) // make connection slow down
+		return
+	}
+
 	var globalServerConfig = sharedNodeConfig.GlobalServerConfig
 	if globalServerConfig != nil && !globalServerConfig.HTTPAll.SupportsLowVersionHTTP && (rawReq.ProtoMajor < 1 /** 0.x **/ || (rawReq.ProtoMajor == 1 && rawReq.ProtoMinor == 0 /** 1.0 **/)) {
 		http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusBadRequest)
+		time.Sleep(1 * time.Second) // make connection slow down
 		return
 	}
 
 	// 不支持Connect
 	if rawReq.Method == http.MethodConnect {
 		http.Error(rawWriter, "Method Not Allowed", http.StatusMethodNotAllowed)
+		time.Sleep(1 * time.Second) // make connection slow down
 		return
 	}
 
@@ -154,7 +162,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 		domain = reqHost
 	}
 
-	server, serverName := this.findNamedServer(domain)
+	server, serverName := this.findNamedServer(domain, false)
 	if server == nil {
 		if server == nil {
 			// 增加默认的一个服务
@@ -177,6 +185,12 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 					return
 				}
 				clientConn.SetUserId(server.UserId)
+
+				var userPlanId int64
+				if server.UserPlan != nil && server.UserPlan.Id > 0 {
+					userPlanId = server.UserPlan.Id
+				}
+				clientConn.SetUserPlanId(userPlanId)
 			}
 		}
 	}

internal/nodes/listener_tcp.go

@@ -80,6 +80,12 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
 			return nil
 		}
 		clientConn.SetUserId(server.UserId)
+
+		var userPlanId int64
+		if server.UserPlan != nil && server.UserPlan.Id > 0 {
+			userPlanId = server.UserPlan.Id
+		}
+		clientConn.SetUserPlanId(userPlanId)
 	} else {
 		tlsConn, ok := conn.(*tls.Conn)
 		if ok {
@@ -92,6 +98,12 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
 						return nil
 					}
 					clientConn.SetUserId(server.UserId)
+
+					var userPlanId int64
+					if server.UserPlan != nil && server.UserPlan.Id > 0 {
+						userPlanId = server.UserPlan.Id
+					}
+					clientConn.SetUserPlanId(userPlanId)
 				}
 			}
 		}

internal/nodes/listener_udp.go

@@ -404,7 +404,11 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyListener
 					stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
 
 					// 带宽
-					stats.SharedBandwidthStatManager.AddBandwidth(server.UserId, server.Id, int64(n), int64(n))
+					var userPlanId int64
+					if server.UserPlan != nil && server.UserPlan.Id > 0 {
+						userPlanId = server.UserPlan.Id
+					}
+					stats.SharedBandwidthStatManager.AddBandwidth(server.UserId, userPlanId, server.Id, int64(n), int64(n))
 				}
 			}
 			if err != nil {

internal/nodes/node.go

@@ -139,9 +139,6 @@ func (this *Node) Start() {
 		remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
 	}
 
-	// 调整系统参数
-	this.checkSystem()
-
 	// 启动事件
 	events.Notify(events.EventStart)
 
@@ -149,12 +146,12 @@ func (this *Node) Start() {
 	remotelogs.Println("NODE", "init config ...")
 	err = this.syncConfig(0)
 	if err != nil {
-		_, err := nodeconfigs.SharedNodeConfig()
+		_, err = nodeconfigs.SharedNodeConfig()
 		if err != nil {
 			// 无本地数据时，会尝试多次读取
 			tryTimes := 0
 			for {
-				err := this.syncConfig(0)
+				err = this.syncConfig(0)
 				if err != nil {
 					tryTimes++
 
@@ -208,6 +205,9 @@ func (this *Node) Start() {
 	sharedNodeConfig = nodeConfig
 	this.onReload(nodeConfig, true)
 
+	// 调整系统参数
+	go this.tuneSystemParameters()
+
 	// 发送事件
 	events.Notify(events.EventLoaded)
 
@@ -808,6 +808,36 @@ func (this *Node) listenSock() error {
 				_ = cmd.Reply(&gosock.Command{Params: maps.Map{
 					"stats": m,
 				}})
+			case "cache.garbage":
+				var shouldDelete = maps.NewMap(cmd.Params).GetBool("delete")
+
+				var count = 0
+				var sampleFiles = []string{}
+				err := caches.SharedManager.ScanGarbageCaches(func(path string) error {
+					count++
+					if len(sampleFiles) < 10 {
+						sampleFiles = append(sampleFiles, path)
+					}
+
+					if shouldDelete {
+						_ = os.Remove(path)                               // .cache
+						_ = os.Remove(caches.PartialRangesFilePath(path)) // @range.cache
+					}
+
+					return nil
+				})
+				if err != nil {
+					_ = cmd.Reply(&gosock.Command{Params: maps.Map{
+						"isOk":  false,
+						"error": err.Error(),
+					}})
+				} else {
+					_ = cmd.Reply(&gosock.Command{Params: maps.Map{
+						"isOk":        true,
+						"count":       count,
+						"sampleFiles": sampleFiles,
+					}})
+				}
 			}
 		})
 
@@ -1048,11 +1078,15 @@ func (this *Node) reloadServer() {
 }
 
 // 检查系统
-func (this *Node) checkSystem() {
+func (this *Node) tuneSystemParameters() {
 	if runtime.GOOS != "linux" || os.Getgid() != 0 {
 		return
 	}
 
+	if sharedNodeConfig == nil || !sharedNodeConfig.AutoSystemTuning {
+		return
+	}
+
 	type variable struct {
 		name     string
 		minValue int
@@ -1061,7 +1095,8 @@ func (this *Node) checkSystem() {
 
 	const dir = "/proc/sys"
 
-	for _, v := range []variable{
+	// net
+	var systemParameters = []variable{
 		{name: "net.core.somaxconn", minValue: 2048},
 		{name: "net.ipv4.tcp_max_syn_backlog", minValue: 2048},
 		{name: "net.core.netdev_max_backlog", minValue: 4096},
@@ -1071,7 +1106,28 @@ func (this *Node) checkSystem() {
 		{name: "net.core.wmem_default", minValue: 4 << 20},
 		{name: "net.core.rmem_max", minValue: 32 << 20},
 		{name: "net.core.wmem_max", minValue: 32 << 20},
-	} {
+	}
+
+	// vm
+	var systemMemory = utils.SystemMemoryGB()
+	if systemMemory >= 128 {
+		systemParameters = append(systemParameters, []variable{
+			{name: "vm.dirty_background_ratio", minValue: 40},
+			{name: "vm.dirty_ratio", minValue: 60},
+		}...)
+	} else if systemMemory >= 64 {
+		systemParameters = append(systemParameters, []variable{
+			{name: "vm.dirty_background_ratio", minValue: 30},
+			{name: "vm.dirty_ratio", minValue: 50},
+		}...)
+	} else if systemMemory >= 16 {
+		systemParameters = append(systemParameters, []variable{
+			{name: "vm.dirty_background_ratio", minValue: 15},
+			{name: "vm.dirty_ratio", minValue: 30},
+		}...)
+	}
+
+	for _, v := range systemParameters {
 		var path = dir + "/" + strings.Replace(v.name, ".", "/", -1)
 		data, err := os.ReadFile(path)
 		if err != nil {

internal/nodes/node_status_executor.go

@@ -223,6 +223,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 	// 当前TeaWeb所在的fs
 	var rootFS = ""
 	var rootTotal = uint64(0)
+	var totalUsed = uint64(0)
 	if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) {
 		for _, p := range partitions {
 			if p.Mountpoint == "/" {
@@ -230,6 +231,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 				usage, _ := disk.Usage(p.Mountpoint)
 				if usage != nil {
 					rootTotal = usage.Total
+					totalUsed = usage.Used
 				}
 				break
 			}
@@ -237,7 +239,6 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 	}
 
 	var total = rootTotal
-	var totalUsage = uint64(0)
 	var maxUsage = float64(0)
 	for _, partition := range partitions {
 		if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") {
@@ -256,15 +257,17 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 
 		if partition.Mountpoint != "/" && (usage.Total != rootTotal || total == 0) {
 			total += usage.Total
-		}
-		totalUsage += usage.Used
-		if usage.UsedPercent >= maxUsage {
-			maxUsage = usage.UsedPercent
-			status.DiskMaxUsagePartition = partition.Mountpoint
+			totalUsed += usage.Used
+			if usage.UsedPercent >= maxUsage {
+				maxUsage = usage.UsedPercent
+				status.DiskMaxUsagePartition = partition.Mountpoint
+			}
 		}
 	}
 	status.DiskTotal = total
-	status.DiskUsage = float64(totalUsage) / float64(total)
+	if total > 0 {
+		status.DiskUsage = float64(totalUsed) / float64(total)
+	}
 	status.DiskMaxUsage = maxUsage / 100
 
 	// 记录监控数据
@@ -280,7 +283,7 @@ func (this *NodeStatusExecutor) updateCacheSpace(status *nodeconfigs.NodeStatus)
 	var result = []maps.Map{}
 	var cachePaths = caches.SharedManager.FindAllCachePaths()
 	for _, path := range cachePaths {
-		stat, err := fsutils.Stat(path)
+		stat, err := fsutils.StatDevice(path)
 		if err != nil {
 			return
 		}

internal/nodes/node_tasks.go

@@ -4,6 +4,7 @@ package nodes
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
@@ -16,9 +17,13 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
 	"github.com/TeaOSLab/EdgeNode/internal/trackers"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/waf"
 	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/types"
 	"os"
+	"strings"
 	"time"
 )
 
@@ -93,8 +98,15 @@ func (this *Node) execTask(rpcClient *rpc.RPCClient, task *pb.NodeTask) error {
 		err = this.notifyPlusChange()
 	case "toaChanged":
 		err = this.execTOAChangedTask()
+	case "networkSecurityPolicyChanged":
+		err = this.execNetworkSecurityPolicyChangedTask(rpcClient)
 	default:
-		remotelogs.Error("NODE", "task '"+types.String(task.Id)+"', type '"+task.Type+"' has not been handled")
+		// 特殊任务
+		if strings.HasPrefix(task.Type, "ipListDeleted") { // 删除IP名单
+			err = this.execDeleteIPList(task.Type)
+		} else { // 未处理的任务
+			remotelogs.Error("NODE", "task '"+types.String(task.Id)+"', type '"+task.Type+"' has not been handled")
+		}
 	}
 
 	return err
@@ -285,6 +297,34 @@ func (this *Node) execUpdatingServersTask(rpcClient *rpc.RPCClient) error {
 	return nil
 }
 
+// 删除IP名单
+func (this *Node) execDeleteIPList(taskType string) error {
+	optionsString, ok := utils.CutPrefix(taskType, "ipListDeleted@")
+	if !ok {
+		return errors.New("invalid task type '" + taskType + "'")
+	}
+	var optionMap = maps.Map{}
+	err := json.Unmarshal([]byte(optionsString), &optionMap)
+	if err != nil {
+		return fmt.Errorf("decode options failed: %w, options: %s", err, optionsString)
+	}
+	var listId = optionMap.GetInt64("listId")
+	if listId <= 0 {
+		return nil
+	}
+
+	// 标记已被删除
+	waf.AddDeletedIPList(listId)
+
+	var list = iplibrary.SharedIPListManager.FindList(listId)
+
+	if list != nil {
+		list.SetDeleted()
+	}
+
+	return nil
+}
+
 // 标记任务完成
 func (this *Node) finishTask(taskId int64, taskVersion int64, taskErr error) (success bool) {
 	if taskId <= 0 {

internal/nodes/node_tasks_ext.go

@@ -29,3 +29,8 @@ func (this *Node) execHTTPPagesPolicyChangedTask(rpcClient *rpc.RPCClient) error
 	// stub
 	return nil
 }
+
+func (this *Node) execNetworkSecurityPolicyChangedTask(rpcClient *rpc.RPCClient) error {
+	// stub
+	return nil
+}

internal/stats/bandwidth_stat_manager.go

@@ -62,6 +62,7 @@ type BandwidthStat struct {
 	CountRequests       int64 `json:"countRequests"`
 	CountCachedRequests int64 `json:"countCachedRequests"`
 	CountAttackRequests int64 `json:"countAttackRequests"`
+	UserPlanId          int64 `json:"userPlanId"`
 }
 
 // BandwidthStatManager 服务带宽统计
@@ -153,6 +154,7 @@ func (this *BandwidthStatManager) Loop() error {
 				CountRequests:       stat.CountRequests,
 				CountCachedRequests: stat.CountCachedRequests,
 				CountAttackRequests: stat.CountAttackRequests,
+				UserPlanId:          stat.UserPlanId,
 				NodeRegionId:        regionId,
 			})
 			delete(this.m, key)
@@ -178,7 +180,7 @@ func (this *BandwidthStatManager) Loop() error {
 }
 
 // AddBandwidth 添加带宽数据
-func (this *BandwidthStatManager) AddBandwidth(userId int64, serverId int64, peekBytes int64, totalBytes int64) {
+func (this *BandwidthStatManager) AddBandwidth(userId int64, userPlanId int64, serverId int64, peekBytes int64, totalBytes int64) {
 	if serverId <= 0 || (peekBytes == 0 && totalBytes == 0) {
 		return
 	}
@@ -217,6 +219,7 @@ func (this *BandwidthStatManager) AddBandwidth(userId int64, serverId int64, pee
 			Day:              day,
 			TimeAt:           timeAt,
 			UserId:           userId,
+			UserPlanId:       userPlanId,
 			ServerId:         serverId,
 			CurrentBytes:     peekBytes,
 			MaxBytes:         peekBytes,

internal/stats/bandwidth_stat_manager_test.go

@@ -12,22 +12,22 @@ import (
 
 func TestBandwidthStatManager_Add(t *testing.T) {
 	var manager = stats.NewBandwidthStatManager()
-	manager.AddBandwidth(1, 1, 10, 10)
-	manager.AddBandwidth(1, 1, 10, 10)
-	manager.AddBandwidth(1, 1, 10, 10)
+	manager.AddBandwidth(1, 0, 1, 10, 10)
+	manager.AddBandwidth(1, 0, 1, 10, 10)
+	manager.AddBandwidth(1, 0, 1, 10, 10)
 	time.Sleep(1 * time.Second)
-	manager.AddBandwidth(1, 1, 85, 85)
+	manager.AddBandwidth(1, 0, 1, 85, 85)
 	time.Sleep(1 * time.Second)
-	manager.AddBandwidth(1, 1, 25, 25)
-	manager.AddBandwidth(1, 1, 75, 75)
+	manager.AddBandwidth(1, 0, 1, 25, 25)
+	manager.AddBandwidth(1, 0, 1, 75, 75)
 	manager.Inspect()
 }
 
 func TestBandwidthStatManager_Loop(t *testing.T) {
 	var manager = stats.NewBandwidthStatManager()
-	manager.AddBandwidth(1, 1, 10, 10)
-	manager.AddBandwidth(1, 1, 10, 10)
-	manager.AddBandwidth(1, 1, 10, 10)
+	manager.AddBandwidth(1, 0, 1, 10, 10)
+	manager.AddBandwidth(1, 0, 1, 10, 10)
+	manager.AddBandwidth(1, 0, 1, 10, 10)
 	err := manager.Loop()
 	if err != nil {
 		t.Fatal(err)
@@ -40,7 +40,7 @@ func BenchmarkBandwidthStatManager_Add(b *testing.B) {
 		var i int
 		for pb.Next() {
 			i++
-			manager.AddBandwidth(1, int64(i%100), 10, 10)
+			manager.AddBandwidth(1, 0, int64(i%100), 10, 10)
 		}
 	})
 }

internal/stats/traffic_stat_manager.go

@@ -142,24 +142,26 @@ func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string,
 	item.PlanId = planId
 
 	// 单个域名流量
-	var domainKey = types.String(timestamp) + "@" + domain
-	serverDomainMap, ok := this.domainsMap[serverId]
-	if !ok {
-		serverDomainMap = map[string]*TrafficItem{}
-		this.domainsMap[serverId] = serverDomainMap
-	}
+	if len(domain) < 128 {
+		var domainKey = types.String(timestamp) + "@" + domain
+		serverDomainMap, ok := this.domainsMap[serverId]
+		if !ok {
+			serverDomainMap = map[string]*TrafficItem{}
+			this.domainsMap[serverId] = serverDomainMap
+		}
 
-	domainItem, ok := serverDomainMap[domainKey]
-	if !ok {
-		domainItem = &TrafficItem{}
-		serverDomainMap[domainKey] = domainItem
+		domainItem, ok := serverDomainMap[domainKey]
+		if !ok {
+			domainItem = &TrafficItem{}
+			serverDomainMap[domainKey] = domainItem
+		}
+		domainItem.Bytes += bytes
+		domainItem.CachedBytes += cachedBytes
+		domainItem.CountRequests += countRequests
+		domainItem.CountCachedRequests += countCachedRequests
+		domainItem.CountAttackRequests += countAttacks
+		domainItem.AttackBytes += attackBytes
 	}
-	domainItem.Bytes += bytes
-	domainItem.CachedBytes += cachedBytes
-	domainItem.CountRequests += countRequests
-	domainItem.CountCachedRequests += countCachedRequests
-	domainItem.CountAttackRequests += countAttacks
-	domainItem.AttackBytes += attackBytes
 
 	this.locker.Unlock()
 }

internal/ttlcache/cache.go

@@ -3,9 +3,10 @@ package ttlcache
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"runtime"
 )
 
-var SharedCache = NewBigCache()
+var SharedInt64Cache = NewBigCache[int64]()
 
 // Cache TTL缓存
 // 最大的缓存时间为30 * 86400
@@ -13,24 +14,25 @@ var SharedCache = NewBigCache()
 //
 //	    Piece1            |  Piece2 | Piece3 | ...
 //	[ Item1, Item2, ... ] |   ...
-type Cache struct {
+type Cache[T any] struct {
 	isDestroyed bool
-	pieces      []*Piece
+	pieces      []*Piece[T]
 	countPieces uint64
 	maxItems    int
 
-	gcPieceIndex int
+	maxPiecesPerGC int
+	gcPieceIndex   int
 }
 
-func NewBigCache() *Cache {
+func NewBigCache[T any]() *Cache[T] {
 	var delta = utils.SystemMemoryGB() / 2
 	if delta <= 0 {
 		delta = 1
 	}
-	return NewCache(NewMaxItemsOption(delta * 1_000_000))
+	return NewCache[T](NewMaxItemsOption(delta * 1_000_000))
 }
 
-func NewCache(opt ...OptionInterface) *Cache {
+func NewCache[T any](opt ...OptionInterface) *Cache[T] {
 	var countPieces = 256
 	var maxItems = 1_000_000
 
@@ -61,13 +63,20 @@ func NewCache(opt ...OptionInterface) *Cache {
 		}
 	}
 
-	var cache = &Cache{
-		countPieces: uint64(countPieces),
-		maxItems:    maxItems,
+	var maxPiecesPerGC = 4
+	var numCPU = runtime.NumCPU() / 2
+	if numCPU > maxPiecesPerGC {
+		maxPiecesPerGC = numCPU
+	}
+
+	var cache = &Cache[T]{
+		countPieces:    uint64(countPieces),
+		maxItems:       maxItems,
+		maxPiecesPerGC: maxPiecesPerGC,
 	}
 
 	for i := 0; i < countPieces; i++ {
-		cache.pieces = append(cache.pieces, NewPiece(maxItems/countPieces))
+		cache.pieces = append(cache.pieces, NewPiece[T](maxItems/countPieces))
 	}
 
 	// Add to manager
@@ -76,7 +85,7 @@ func NewCache(opt ...OptionInterface) *Cache {
 	return cache
 }
 
-func (this *Cache) Write(key string, value any, expiredAt int64) (ok bool) {
+func (this *Cache[T]) Write(key string, value T, expiredAt int64) (ok bool) {
 	if this.isDestroyed {
 		return
 	}
@@ -92,20 +101,20 @@ func (this *Cache) Write(key string, value any, expiredAt int64) (ok bool) {
 	}
 	var uint64Key = HashKey([]byte(key))
 	var pieceIndex = uint64Key % this.countPieces
-	return this.pieces[pieceIndex].Add(uint64Key, &Item{
+	return this.pieces[pieceIndex].Add(uint64Key, &Item[T]{
 		Value:     value,
 		expiredAt: expiredAt,
 	})
 }
 
-func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64, extend bool) int64 {
+func (this *Cache[T]) IncreaseInt64(key string, delta T, expiredAt int64, extend bool) T {
 	if this.isDestroyed {
-		return 0
+		return any(0).(T)
 	}
 
 	var currentTimestamp = fasttime.Now().Unix()
 	if expiredAt <= currentTimestamp {
-		return 0
+		return any(0).(T)
 	}
 
 	var maxExpiredAt = currentTimestamp + 30*86400
@@ -117,47 +126,47 @@ func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64, exten
 	return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiredAt, extend)
 }
 
-func (this *Cache) Read(key string) (item *Item) {
+func (this *Cache[T]) Read(key string) (item *Item[T]) {
 	var uint64Key = HashKey([]byte(key))
 	return this.pieces[uint64Key%this.countPieces].Read(uint64Key)
 }
 
-func (this *Cache) Delete(key string) {
+func (this *Cache[T]) Delete(key string) {
 	var uint64Key = HashKey([]byte(key))
 	this.pieces[uint64Key%this.countPieces].Delete(uint64Key)
 }
 
-func (this *Cache) Count() (count int) {
+func (this *Cache[T]) Count() (count int) {
 	for _, piece := range this.pieces {
 		count += piece.Count()
 	}
 	return
 }
 
-func (this *Cache) GC() {
+func (this *Cache[T]) GC() {
 	var index = this.gcPieceIndex
-	const maxPiecesPerGC = 4
-	for i := index; i < index+maxPiecesPerGC; i++ {
+
+	for i := index; i < index+this.maxPiecesPerGC; i++ {
 		if i >= int(this.countPieces) {
 			break
 		}
 		this.pieces[i].GC()
 	}
 
-	index += maxPiecesPerGC
+	index += this.maxPiecesPerGC
 	if index >= int(this.countPieces) {
 		index = 0
 	}
 	this.gcPieceIndex = index
 }
 
-func (this *Cache) Clean() {
+func (this *Cache[T]) Clean() {
 	for _, piece := range this.pieces {
 		piece.Clean()
 	}
 }
 
-func (this *Cache) Destroy() {
+func (this *Cache[T]) Destroy() {
 	SharedManager.Remove(this)
 
 	this.isDestroyed = true

internal/ttlcache/cache_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"runtime"
 	"strconv"
 	"sync/atomic"
@@ -14,7 +15,7 @@ import (
 )
 
 func TestNewCache(t *testing.T) {
-	var cache = NewCache()
+	var cache = NewCache[int]()
 	cache.Write("a", 1, time.Now().Unix()+3600)
 	cache.Write("b", 2, time.Now().Unix()+1)
 	cache.Write("c", 1, time.Now().Unix()+3602)
@@ -28,7 +29,9 @@ func TestNewCache(t *testing.T) {
 		}
 	}
 	t.Log("a:", cache.Read("a"))
-	time.Sleep(5 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(5 * time.Second)
+	}
 
 	for i := 0; i < len(cache.pieces); i++ {
 		cache.GC()
@@ -40,12 +43,19 @@ func TestNewCache(t *testing.T) {
 }
 
 func TestCache_Memory(t *testing.T) {
-	testutils.StartMemoryStats(t)
+	if !testutils.IsSingleTesting() {
+		return
+	}
 
-	var cache = NewCache()
-	var count = 2_000_000
+	var cache = NewCache[int]()
+
+	testutils.StartMemoryStats(t, func() {
+		t.Log(cache.Count(), "items")
+	})
+
+	var count = 20_000_000
 	for i := 0; i < count; i++ {
-		cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+3600)
+		cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+int64(rands.Int(0, 300)))
 	}
 
 	t.Log(cache.Count())
@@ -61,27 +71,27 @@ func TestCache_Memory(t *testing.T) {
 
 	cache.Count()
 
-	time.Sleep(10 * time.Second)
+	time.Sleep(3600 * time.Second)
 }
 
 func TestCache_IncreaseInt64(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var cache = NewCache()
+	var cache = NewCache[int64]()
 	var unixTime = time.Now().Unix()
 
 	{
 		cache.IncreaseInt64("a", 1, unixTime+3600, false)
 		var item = cache.Read("a")
 		t.Log(item)
-		a.IsTrue(item.Value == int64(1))
+		a.IsTrue(item.Value == 1)
 		a.IsTrue(item.expiredAt == unixTime+3600)
 	}
 	{
 		cache.IncreaseInt64("a", 1, unixTime+3600+1, true)
 		var item = cache.Read("a")
 		t.Log(item)
-		a.IsTrue(item.Value == int64(2))
+		a.IsTrue(item.Value == 2)
 		a.IsTrue(item.expiredAt == unixTime+3600+1)
 	}
 	{
@@ -97,7 +107,7 @@ func TestCache_IncreaseInt64(t *testing.T) {
 func TestCache_Read(t *testing.T) {
 	runtime.GOMAXPROCS(1)
 
-	var cache = NewCache(PiecesOption{Count: 32})
+	var cache = NewCache[int](PiecesOption{Count: 32})
 
 	for i := 0; i < 10_000_000; i++ {
 		cache.Write("HELLO_WORLD_"+strconv.Itoa(i), i, time.Now().Unix()+int64(i%10240)+1)
@@ -119,7 +129,11 @@ func TestCache_Read(t *testing.T) {
 }
 
 func TestCache_GC(t *testing.T) {
-	var cache = NewCache(&PiecesOption{Count: 5})
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var cache = NewCache[int](&PiecesOption{Count: 5})
 	cache.Write("a", 1, time.Now().Unix()+1)
 	cache.Write("b", 2, time.Now().Unix()+2)
 	cache.Write("c", 3, time.Now().Unix()+3)
@@ -153,26 +167,30 @@ func TestCache_GC(t *testing.T) {
 }
 
 func TestCache_GC2(t *testing.T) {
-	runtime.GOMAXPROCS(1)
-
-	var cache1 = NewCache(NewPiecesOption(32))
-	for i := 0; i < 1_000_000; i++ {
-		cache1.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(0, 10)))
+	if !testutils.IsSingleTesting() {
+		return
 	}
 
-	var cache2 = NewCache(NewPiecesOption(5))
+	runtime.GOMAXPROCS(1)
+
+	var cache1 = NewCache[int](NewPiecesOption(256))
+	for i := 0; i < 10_000_000; i++ {
+		cache1.Write(strconv.Itoa(i), i, time.Now().Unix()+10)
+	}
+
+	var cache2 = NewCache[int](NewPiecesOption(5))
 	for i := 0; i < 1_000_000; i++ {
 		cache2.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(0, 10)))
 	}
 
-	for i := 0; i < 100; i++ {
-		t.Log(cache1.Count(), "items", cache2.Count(), "items")
+	for i := 0; i < 3600; i++ {
+		t.Log(timeutil.Format("H:i:s"), cache1.Count(), "items", cache2.Count(), "items")
 		time.Sleep(1 * time.Second)
 	}
 }
 
 func TestCacheDestroy(t *testing.T) {
-	var cache = NewCache()
+	var cache = NewCache[int]()
 	t.Log("count:", SharedManager.Count())
 	cache.Destroy()
 	t.Log("count:", SharedManager.Count())
@@ -181,7 +199,7 @@ func TestCacheDestroy(t *testing.T) {
 func BenchmarkNewCache(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var cache = NewCache(NewPiecesOption(128))
+	var cache = NewCache[int](NewPiecesOption(128))
 	for i := 0; i < 2_000_000; i++ {
 		cache.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(10, 100)))
 	}
@@ -199,7 +217,7 @@ func BenchmarkNewCache(b *testing.B) {
 func BenchmarkCache_Add(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var cache = NewCache()
+	var cache = NewCache[int]()
 	for i := 0; i < b.N; i++ {
 		cache.Write(strconv.Itoa(i), i, fasttime.Now().Unix()+int64(i%1024))
 	}
@@ -208,7 +226,7 @@ func BenchmarkCache_Add(b *testing.B) {
 func BenchmarkCache_Add_Parallel(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var cache = NewCache()
+	var cache = NewCache[int64]()
 	var i int64
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
@@ -221,7 +239,7 @@ func BenchmarkCache_Add_Parallel(b *testing.B) {
 func BenchmarkNewCacheGC(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var cache = NewCache(NewPiecesOption(1024))
+	var cache = NewCache[int](NewPiecesOption(1024))
 	for i := 0; i < 3_000_000; i++ {
 		cache.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(0, 100)))
 	}
@@ -238,7 +256,7 @@ func BenchmarkNewCacheGC(b *testing.B) {
 func BenchmarkNewCacheClean(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var cache = NewCache(NewPiecesOption(128))
+	var cache = NewCache[int](NewPiecesOption(128))
 	for i := 0; i < 3_000_000; i++ {
 		cache.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(10, 100)))
 	}

internal/ttlcache/item.go

@@ -1,6 +1,6 @@
 package ttlcache
 
-type Item struct {
-	Value     any
+type Item[T any] struct {
+	Value     T
 	expiredAt int64
 }

internal/ttlcache/manager.go

@@ -11,17 +11,21 @@ import (
 
 var SharedManager = NewManager()
 
+type GCAble interface {
+	GC()
+}
+
 type Manager struct {
 	ticker *time.Ticker
 	locker sync.Mutex
 
-	cacheMap map[*Cache]zero.Zero
+	cacheMap map[GCAble]zero.Zero
 }
 
 func NewManager() *Manager {
 	var manager = &Manager{
 		ticker:   time.NewTicker(2 * time.Second),
-		cacheMap: map[*Cache]zero.Zero{},
+		cacheMap: map[GCAble]zero.Zero{},
 	}
 
 	goman.New(func() {
@@ -41,13 +45,13 @@ func (this *Manager) init() {
 	}
 }
 
-func (this *Manager) Add(cache *Cache) {
+func (this *Manager) Add(cache GCAble) {
 	this.locker.Lock()
 	this.cacheMap[cache] = zero.New()
 	this.locker.Unlock()
 }
 
-func (this *Manager) Remove(cache *Cache) {
+func (this *Manager) Remove(cache GCAble) {
 	this.locker.Lock()
 	delete(this.cacheMap, cache)
 	this.locker.Unlock()

internal/ttlcache/piece.go

@@ -3,12 +3,11 @@ package ttlcache
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/expires"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
-	"github.com/iwind/TeaGo/types"
 	"sync"
 )
 
-type Piece struct {
-	m           map[uint64]*Item
+type Piece[T any] struct {
+	m           map[uint64]*Item[T]
 	expiresList *expires.List
 	maxItems    int
 	lastGCTime  int64
@@ -16,20 +15,28 @@ type Piece struct {
 	locker sync.RWMutex
 }
 
-func NewPiece(maxItems int) *Piece {
-	return &Piece{
-		m:           map[uint64]*Item{},
+func NewPiece[T any](maxItems int) *Piece[T] {
+	return &Piece[T]{
+		m:           map[uint64]*Item[T]{},
 		expiresList: expires.NewSingletonList(),
 		maxItems:    maxItems,
 	}
 }
 
-func (this *Piece) Add(key uint64, item *Item) (ok bool) {
-	this.locker.Lock()
+func (this *Piece[T]) Add(key uint64, item *Item[T]) (ok bool) {
+	this.locker.RLock()
 	if this.maxItems > 0 && len(this.m) >= this.maxItems {
-		this.locker.Unlock()
+		this.locker.RUnlock()
 		return
 	}
+	this.locker.RUnlock()
+
+	this.locker.Lock()
+	oldItem, exists := this.m[key]
+	if exists && oldItem.expiredAt == item.expiredAt {
+		this.locker.Unlock()
+		return true
+	}
 	this.m[key] = item
 	this.locker.Unlock()
 
@@ -38,11 +45,14 @@ func (this *Piece) Add(key uint64, item *Item) (ok bool) {
 	return true
 }
 
-func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, extend bool) (result int64) {
+func (this *Piece[T]) IncreaseInt64(key uint64, delta T, expiredAt int64, extend bool) (result T) {
 	this.locker.Lock()
 	item, ok := this.m[key]
 	if ok && item.expiredAt > fasttime.Now().Unix() {
-		result = types.Int64(item.Value) + delta
+		int64Value, isInt64 := any(item.Value).(int64)
+		if isInt64 {
+			result = any(int64Value + any(delta).(int64)).(T)
+		}
 		item.Value = result
 		if extend {
 			item.expiredAt = expiredAt
@@ -51,7 +61,7 @@ func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, exten
 	} else {
 		if len(this.m) < this.maxItems {
 			result = delta
-			this.m[key] = &Item{
+			this.m[key] = &Item[T]{
 				Value:     delta,
 				expiredAt: expiredAt,
 			}
@@ -63,7 +73,7 @@ func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, exten
 	return
 }
 
-func (this *Piece) Delete(key uint64) {
+func (this *Piece[T]) Delete(key uint64) {
 	this.expiresList.Remove(key)
 
 	this.locker.Lock()
@@ -71,7 +81,7 @@ func (this *Piece) Delete(key uint64) {
 	this.locker.Unlock()
 }
 
-func (this *Piece) Read(key uint64) (item *Item) {
+func (this *Piece[T]) Read(key uint64) (item *Item[T]) {
 	this.locker.RLock()
 	item = this.m[key]
 	if item != nil && item.expiredAt < fasttime.Now().Unix() {
@@ -82,27 +92,27 @@ func (this *Piece) Read(key uint64) (item *Item) {
 	return
 }
 
-func (this *Piece) Count() (count int) {
+func (this *Piece[T]) Count() (count int) {
 	this.locker.RLock()
 	count = len(this.m)
 	this.locker.RUnlock()
 	return
 }
 
-func (this *Piece) GC() {
+func (this *Piece[T]) GC() {
 	var currentTime = fasttime.Now().Unix()
 	if this.lastGCTime == 0 {
 		this.lastGCTime = currentTime - 3600
 	}
 
-	var min = this.lastGCTime
-	var max = currentTime
-	if min > max {
+	var minTime = this.lastGCTime
+	var maxTime = currentTime
+	if minTime > maxTime {
 		// 过去的时间比现在大，则从这一秒重新开始
-		min = max
+		minTime = maxTime
 	}
 
-	for i := min; i <= max; i++ {
+	for i := minTime; i <= maxTime; i++ {
 		var itemMap = this.expiresList.GC(i)
 		if len(itemMap) > 0 {
 			this.gcItemMap(itemMap)
@@ -112,15 +122,15 @@ func (this *Piece) GC() {
 	this.lastGCTime = currentTime
 }
 
-func (this *Piece) Clean() {
+func (this *Piece[T]) Clean() {
 	this.locker.Lock()
-	this.m = map[uint64]*Item{}
+	this.m = map[uint64]*Item[T]{}
 	this.locker.Unlock()
 
 	this.expiresList.Clean()
 }
 
-func (this *Piece) Destroy() {
+func (this *Piece[T]) Destroy() {
 	this.locker.Lock()
 	this.m = nil
 	this.locker.Unlock()
@@ -128,7 +138,7 @@ func (this *Piece) Destroy() {
 	this.expiresList.Clean()
 }
 
-func (this *Piece) gcItemMap(itemMap expires.ItemMap) {
+func (this *Piece[T]) gcItemMap(itemMap expires.ItemMap) {
 	this.locker.Lock()
 	for key := range itemMap {
 		delete(this.m, key)

internal/ttlcache/piece_test.go

@@ -7,10 +7,10 @@ import (
 )
 
 func TestPiece_Add(t *testing.T) {
-	piece := NewPiece(10)
-	piece.Add(1, &Item{expiredAt: time.Now().Unix() + 3600})
-	piece.Add(2, &Item{})
-	piece.Add(3, &Item{})
+	piece := NewPiece[int](10)
+	piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600})
+	piece.Add(2, &Item[int]{})
+	piece.Add(3, &Item[int]{})
 	piece.Delete(3)
 	for key, item := range piece.m {
 		t.Log(key, item.Value)
@@ -18,19 +18,29 @@ func TestPiece_Add(t *testing.T) {
 	t.Log(piece.Read(1))
 }
 
+func TestPiece_Add_Same(t *testing.T) {
+	piece := NewPiece[int](10)
+	piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600})
+	piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600})
+	for key, item := range piece.m {
+		t.Log(key, item.Value)
+	}
+	t.Log(piece.Read(1))
+}
+
 func TestPiece_MaxItems(t *testing.T) {
-	piece := NewPiece(10)
+	piece := NewPiece[int](10)
 	for i := 0; i < 1000; i++ {
-		piece.Add(uint64(i), &Item{expiredAt: time.Now().Unix() + 3600})
+		piece.Add(uint64(i), &Item[int]{expiredAt: time.Now().Unix() + 3600})
 	}
 	t.Log(len(piece.m))
 }
 
 func TestPiece_GC(t *testing.T) {
-	piece := NewPiece(10)
-	piece.Add(1, &Item{Value: 1, expiredAt: time.Now().Unix() + 1})
-	piece.Add(2, &Item{Value: 2, expiredAt: time.Now().Unix() + 1})
-	piece.Add(3, &Item{Value: 3, expiredAt: time.Now().Unix() + 1})
+	piece := NewPiece[int](10)
+	piece.Add(1, &Item[int]{Value: 1, expiredAt: time.Now().Unix() + 1})
+	piece.Add(2, &Item[int]{Value: 2, expiredAt: time.Now().Unix() + 1})
+	piece.Add(3, &Item[int]{Value: 3, expiredAt: time.Now().Unix() + 1})
 	t.Log("before gc ===")
 	for key, item := range piece.m {
 		t.Log(key, item.Value)
@@ -46,9 +56,9 @@ func TestPiece_GC(t *testing.T) {
 }
 
 func TestPiece_GC2(t *testing.T) {
-	piece := NewPiece(10)
+	piece := NewPiece[int](10)
 	for i := 0; i < 10_000; i++ {
-		piece.Add(uint64(i), &Item{Value: 1, expiredAt: time.Now().Unix() + int64(rands.Int(1, 10))})
+		piece.Add(uint64(i), &Item[int]{Value: 1, expiredAt: time.Now().Unix() + int64(rands.Int(1, 10))})
 	}
 
 	time.Sleep(1 * time.Second)

internal/utils/cachehits/stat.go

@@ -0,0 +1,161 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package cachehits
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const countSamples = 100_000
+
+type Item struct {
+	countHits   uint64
+	countCached uint64
+	timestamp   int64
+
+	isGood bool
+	isBad  bool
+}
+
+type Stat struct {
+	goodRatio uint64
+	maxItems  int
+
+	itemMap map[string]*Item // category => *Item
+	mu      *sync.RWMutex
+
+	ticker *time.Ticker
+}
+
+func NewStat(goodRatio uint64) *Stat {
+	if goodRatio == 0 {
+		goodRatio = 5
+	}
+
+	var maxItems = utils.SystemMemoryGB() * 10_000
+	if maxItems <= 0 {
+		maxItems = 100_000
+	}
+
+	var stat = &Stat{
+		goodRatio: goodRatio,
+		itemMap:   map[string]*Item{},
+		mu:        &sync.RWMutex{},
+		ticker:    time.NewTicker(24 * time.Hour),
+		maxItems:  maxItems,
+	}
+
+	goman.New(func() {
+		stat.init()
+	})
+	return stat
+}
+
+func (this *Stat) init() {
+	for range this.ticker.C {
+		var currentTime = fasttime.Now().Unix()
+
+		this.mu.RLock()
+		for _, item := range this.itemMap {
+			if item.timestamp < currentTime-7*24*86400 {
+				// reset
+				item.countHits = 0
+				item.countCached = 1
+				item.timestamp = currentTime
+				item.isGood = false
+				item.isBad = false
+			}
+		}
+		this.mu.RUnlock()
+	}
+}
+
+func (this *Stat) IncreaseCached(category string) {
+	this.mu.RLock()
+	var item = this.itemMap[category]
+	if item != nil {
+		if item.isGood || item.isBad {
+			this.mu.RUnlock()
+			return
+		}
+
+		atomic.AddUint64(&item.countCached, 1)
+		this.mu.RUnlock()
+		return
+	}
+	this.mu.RUnlock()
+
+	this.mu.Lock()
+
+	if len(this.itemMap) > this.maxItems {
+		// remove one randomly
+		for k := range this.itemMap {
+			delete(this.itemMap, k)
+			break
+		}
+	}
+
+	this.itemMap[category] = &Item{
+		countHits:   0,
+		countCached: 1,
+		timestamp:   fasttime.Now().Unix(),
+	}
+	this.mu.Unlock()
+}
+
+func (this *Stat) IncreaseHit(category string) {
+	this.mu.RLock()
+	defer this.mu.RUnlock()
+
+	var item = this.itemMap[category]
+	if item != nil {
+		if item.isGood || item.isBad {
+			return
+		}
+
+		atomic.AddUint64(&item.countHits, 1)
+		return
+	}
+}
+
+func (this *Stat) IsGood(category string) bool {
+	this.mu.RLock()
+	defer func() {
+		this.mu.RUnlock()
+	}()
+
+	var item = this.itemMap[category]
+	if item != nil {
+		if item.isBad {
+			return false
+		}
+		if item.isGood {
+			return true
+		}
+
+		if item.countCached > countSamples && item.timestamp < fasttime.Now().Unix()-600 /** 10 minutes ago **/ {
+			var isGood = item.countHits*100/item.countCached >= this.goodRatio
+			if isGood {
+				item.isGood = true
+			} else {
+				item.isBad = true
+			}
+
+			return isGood
+		}
+	}
+
+	return true
+}
+
+func (this *Stat) Len() int {
+	this.mu.RLock()
+	defer this.mu.RUnlock()
+
+	return len(this.itemMap)
+}

internal/utils/cachehits/stat_test.go

@@ -0,0 +1,107 @@
+// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package cachehits_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/cachehits"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/assert"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	"runtime"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestNewStat(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	{
+		var stat = cachehits.NewStat(20)
+		for i := 0; i < 1000; i++ {
+			stat.IncreaseCached("a")
+		}
+
+		a.IsTrue(stat.IsGood("a"))
+	}
+
+	{
+		var stat = cachehits.NewStat(5)
+		for i := 0; i < 10000; i++ {
+			stat.IncreaseCached("a")
+		}
+		for i := 0; i < 500; i++ {
+			stat.IncreaseHit("a")
+		}
+
+		stat.IncreaseHit("b") // empty
+
+		a.IsTrue(stat.IsGood("a"))
+		a.IsTrue(stat.IsGood("b"))
+	}
+
+	{
+		var stat = cachehits.NewStat(10)
+		for i := 0; i < 10000; i++ {
+			stat.IncreaseCached("a")
+		}
+		for i := 0; i < 1000; i++ {
+			stat.IncreaseHit("a")
+		}
+
+		stat.IncreaseHit("b") // empty
+
+		a.IsTrue(stat.IsGood("a"))
+		a.IsTrue(stat.IsGood("b"))
+	}
+
+	{
+		var stat = cachehits.NewStat(5)
+		for i := 0; i < 10001; i++ {
+			stat.IncreaseCached("a")
+		}
+		for i := 0; i < 499; i++ {
+			stat.IncreaseHit("a")
+		}
+
+		a.IsFalse(stat.IsGood("a"))
+	}
+}
+
+func TestNewStat_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var stat = cachehits.NewStat(20)
+	for i := 0; i < 10_000_000; i++ {
+		stat.IncreaseCached("a" + types.String(i))
+	}
+
+	time.Sleep(60 * time.Second)
+
+	t.Log(stat.Len())
+}
+
+func BenchmarkStat(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var stat = cachehits.NewStat(5)
+	for i := 0; i < 1_000_000; i++ {
+		stat.IncreaseCached("a" + types.String(i))
+	}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var key = strconv.Itoa(rands.Int(0, 100_000))
+			stat.IncreaseCached(key)
+			if rands.Int(0, 3) == 0 {
+				stat.IncreaseHit(key)
+			}
+			_ = stat.IsGood(key)
+		}
+	})
+}

internal/utils/counters/counter.go

@@ -3,18 +3,26 @@
 package counters
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync"
 	"github.com/cespare/xxhash"
-	"runtime"
 	"sync"
 	"time"
 )
 
-type Counter struct {
+const maxItemsPerGroup = 50_000
+
+var SharedCounter = NewCounter[uint32]().WithGC()
+
+type SupportedUIntType interface {
+	uint32 | uint64
+}
+
+type Counter[T SupportedUIntType] struct {
 	countMaps uint64
 	locker    *syncutils.RWMutex
-	itemMaps  []map[uint64]*Item
+	itemMaps  []map[uint64]*Item[T]
 
 	gcTicker *time.Ticker
 	gcIndex  int
@@ -22,20 +30,18 @@ type Counter struct {
 }
 
 // NewCounter create new counter
-func NewCounter() *Counter {
-	var count = runtime.NumCPU() * 4
+func NewCounter[T SupportedUIntType]() *Counter[T] {
+	var count = utils.SystemMemoryGB() * 8
 	if count < 8 {
 		count = 8
-	} else if count > 128 {
-		count = 128
 	}
 
-	var itemMaps = []map[uint64]*Item{}
+	var itemMaps = []map[uint64]*Item[T]{}
 	for i := 0; i < count; i++ {
-		itemMaps = append(itemMaps, map[uint64]*Item{})
+		itemMaps = append(itemMaps, map[uint64]*Item[T]{})
 	}
 
-	var counter = &Counter{
+	var counter = &Counter[T]{
 		countMaps: uint64(count),
 		locker:    syncutils.NewRWMutex(count),
 		itemMaps:  itemMaps,
@@ -45,7 +51,7 @@ func NewCounter() *Counter {
 }
 
 // WithGC start the counter with gc automatically
-func (this *Counter) WithGC() *Counter {
+func (this *Counter[T]) WithGC() *Counter[T] {
 	if this.gcTicker != nil {
 		return this
 	}
@@ -60,23 +66,17 @@ func (this *Counter) WithGC() *Counter {
 }
 
 // Increase key
-func (this *Counter) Increase(key uint64, lifeSeconds int) uint64 {
+func (this *Counter[T]) Increase(key uint64, lifeSeconds int) T {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
 	var item = this.itemMaps[index][key]
 	this.locker.RUnlock(index)
-	if item == nil { // no need to care about duplication
-		item = NewItem(lifeSeconds)
+	if item == nil {
+		// no need to care about duplication
+		// always insert new item even when itemMap is full
+		item = NewItem[T](lifeSeconds)
 		this.locker.Lock(index)
-
-		// check again
-		oldItem, ok := this.itemMaps[index][key]
-		if !ok {
-			this.itemMaps[index][key] = item
-		} else {
-			item = oldItem
-		}
-
+		this.itemMaps[index][key] = item
 		this.locker.Unlock(index)
 	}
 
@@ -87,12 +87,12 @@ func (this *Counter) Increase(key uint64, lifeSeconds int) uint64 {
 }
 
 // IncreaseKey increase string key
-func (this *Counter) IncreaseKey(key string, lifeSeconds int) uint64 {
+func (this *Counter[T]) IncreaseKey(key string, lifeSeconds int) T {
 	return this.Increase(this.hash(key), lifeSeconds)
 }
 
 // Get value of key
-func (this *Counter) Get(key uint64) uint64 {
+func (this *Counter[T]) Get(key uint64) T {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
 	defer this.locker.RUnlock(index)
@@ -104,12 +104,12 @@ func (this *Counter) Get(key uint64) uint64 {
 }
 
 // GetKey get value of string key
-func (this *Counter) GetKey(key string) uint64 {
+func (this *Counter[T]) GetKey(key string) T {
 	return this.Get(this.hash(key))
 }
 
 // Reset key
-func (this *Counter) Reset(key uint64) {
+func (this *Counter[T]) Reset(key uint64) {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
 	var item = this.itemMaps[index][key]
@@ -123,12 +123,12 @@ func (this *Counter) Reset(key uint64) {
 }
 
 // ResetKey string key
-func (this *Counter) ResetKey(key string) {
+func (this *Counter[T]) ResetKey(key string) {
 	this.Reset(this.hash(key))
 }
 
 // TotalItems get items count
-func (this *Counter) TotalItems() int {
+func (this *Counter[T]) TotalItems() int {
 	var total = 0
 
 	for i := 0; i < int(this.countMaps); i++ {
@@ -141,7 +141,7 @@ func (this *Counter) TotalItems() int {
 }
 
 // GC garbage expired items
-func (this *Counter) GC() {
+func (this *Counter[T]) GC() {
 	this.gcLocker.Lock()
 	var gcIndex = this.gcIndex
 
@@ -162,18 +162,39 @@ func (this *Counter) GC() {
 			expiredKeys = append(expiredKeys, key)
 		}
 	}
+	var tooManyItems = len(itemMap) > maxItemsPerGroup // prevent too many items
 	this.locker.RUnlock(gcIndex)
 
 	if len(expiredKeys) > 0 {
+		this.locker.Lock(gcIndex)
 		for _, key := range expiredKeys {
-			this.locker.Lock(gcIndex)
 			delete(itemMap, key)
-			this.locker.Unlock(gcIndex)
 		}
+		this.locker.Unlock(gcIndex)
+	}
+
+	if tooManyItems {
+		this.locker.Lock(gcIndex)
+		var count = len(itemMap) - maxItemsPerGroup
+		if count > 0 {
+			itemMap = this.itemMaps[gcIndex]
+			for key := range itemMap {
+				delete(itemMap, key)
+				count--
+				if count < 0 {
+					break
+				}
+			}
+		}
+		this.locker.Unlock(gcIndex)
 	}
 }
 
+func (this *Counter[T]) CountMaps() int {
+	return int(this.countMaps)
+}
+
 // calculate hash of the key
-func (this *Counter) hash(key string) uint64 {
+func (this *Counter[T]) hash(key string) uint64 {
 	return xxhash.Sum64String(key)
 }

internal/utils/counters/counter_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"runtime"
+	"runtime/debug"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -18,7 +19,7 @@ import (
 func TestCounter_Increase(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	a.IsTrue(counter.Increase(1, 10) == 1)
 	a.IsTrue(counter.Increase(1, 10) == 2)
 	a.IsTrue(counter.Increase(2, 10) == 1)
@@ -31,7 +32,7 @@ func TestCounter_Increase(t *testing.T) {
 func TestCounter_IncreaseKey(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	a.IsTrue(counter.IncreaseKey("1", 10) == 1)
 	a.IsTrue(counter.IncreaseKey("1", 10) == 2)
 	a.IsTrue(counter.IncreaseKey("2", 10) == 1)
@@ -46,7 +47,7 @@ func TestCounter_GC(t *testing.T) {
 		return
 	}
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	counter.Increase(1, 20)
 	time.Sleep(1 * time.Second)
 	counter.Increase(1, 20)
@@ -60,7 +61,7 @@ func TestCounter_GC2(t *testing.T) {
 		return
 	}
 
-	var counter = counters.NewCounter().WithGC()
+	var counter = counters.NewCounter[uint32]().WithGC()
 	for i := 0; i < 1e5; i++ {
 		counter.Increase(uint64(i), rands.Int(10, 300))
 	}
@@ -78,20 +79,26 @@ func TestCounterMemory(t *testing.T) {
 	var stat = &runtime.MemStats{}
 	runtime.ReadMemStats(stat)
 
-	var counter = counters.NewCounter()
-	for i := 0; i < 1e5; i++ {
+	var counter = counters.NewCounter[uint32]()
+	for i := 0; i < 1_000_000; i++ {
 		counter.Increase(uint64(i), rands.Int(10, 300))
 	}
 
+	runtime.GC()
+	runtime.GC()
+	debug.FreeOSMemory()
+
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 	t.Log((stat1.TotalAlloc-stat.TotalAlloc)/(1<<20), "MB")
+
+	t.Log(counter.TotalItems())
 }
 
 func BenchmarkCounter_Increase(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	b.ResetTimer()
 
 	var i uint64
@@ -107,7 +114,7 @@ func BenchmarkCounter_Increase(b *testing.B) {
 func BenchmarkCounter_IncreaseKey(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 
 	go func() {
 		var ticker = time.NewTicker(100 * time.Millisecond)
@@ -131,7 +138,7 @@ func BenchmarkCounter_IncreaseKey(b *testing.B) {
 func BenchmarkCounter_IncreaseKey2(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 
 	go func() {
 		var ticker = time.NewTicker(1 * time.Millisecond)
@@ -155,7 +162,7 @@ func BenchmarkCounter_IncreaseKey2(b *testing.B) {
 func BenchmarkCounter_GC(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 
 	for i := uint64(0); i < 1e5; i++ {
 		counter.IncreaseKey(types.String(i), 20)

internal/utils/counters/item.go

@@ -6,16 +6,16 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 )
 
-type Item struct {
-	lifeSeconds int64
-
-	spanSeconds int64
-	spans       []*Span
+const spanMaxValue = 10_000_000
 
+type Item[T SupportedUIntType] struct {
+	spans          []T
 	lastUpdateTime int64
+	lifeSeconds    int64
+	spanSeconds    int64
 }
 
-func NewItem(lifeSeconds int) *Item {
+func NewItem[T SupportedUIntType](lifeSeconds int) *Item[T] {
 	if lifeSeconds <= 0 {
 		lifeSeconds = 60
 	}
@@ -24,54 +24,100 @@ func NewItem(lifeSeconds int) *Item {
 		spanSeconds = 1
 	}
 	var countSpans = lifeSeconds/spanSeconds + 1 /** prevent index out of bounds **/
-	var spans = []*Span{}
-	for i := 0; i < countSpans; i++ {
-		spans = append(spans, NewSpan())
-	}
 
-	return &Item{
+	return &Item[T]{
 		lifeSeconds:    int64(lifeSeconds),
 		spanSeconds:    int64(spanSeconds),
-		spans:          spans,
+		spans:          make([]T, countSpans),
 		lastUpdateTime: fasttime.Now().Unix(),
 	}
 }
 
-func (this *Item) Increase() uint64 {
+func (this *Item[T]) Increase() (result T) {
 	var currentTime = fasttime.Now().Unix()
-	var spanIndex = int(currentTime % this.lifeSeconds / this.spanSeconds)
-	var span = this.spans[spanIndex]
-	var roundTime = currentTime / this.spanSeconds * this.spanSeconds
+	var currentSpanIndex = this.calculateSpanIndex(currentTime)
 
-	this.lastUpdateTime = currentTime
-
-	if span.Timestamp < roundTime {
-		span.Timestamp = roundTime // update time
-		span.Count = 0             // reset count
+	// return quickly
+	if this.lastUpdateTime == currentTime {
+		if this.spans[currentSpanIndex] < spanMaxValue {
+			this.spans[currentSpanIndex]++
+		}
+		for _, count := range this.spans {
+			result += count
+		}
+		return
 	}
-	span.Count++
 
-	return this.Sum()
-}
+	if this.lastUpdateTime > 0 {
+		if currentTime-this.lastUpdateTime > this.lifeSeconds {
+			for index := range this.spans {
+				this.spans[index] = 0
+			}
+		} else {
+			var lastSpanIndex = this.calculateSpanIndex(this.lastUpdateTime)
 
-func (this *Item) Sum() uint64 {
-	var result uint64 = 0
-	var currentTimestamp = fasttime.Now().Unix()
-	for _, span := range this.spans {
-		if span.Timestamp >= currentTimestamp-this.lifeSeconds {
-			result += span.Count
+			if lastSpanIndex != currentSpanIndex {
+				var countSpans = len(this.spans)
+
+				// reset values between LAST and CURRENT
+				for index := lastSpanIndex + 1; ; index++ {
+					var realIndex = index % countSpans
+					this.spans[realIndex] = 0
+					if realIndex == currentSpanIndex {
+						break
+					}
+				}
+			}
 		}
 	}
+
+	if this.spans[currentSpanIndex] < spanMaxValue {
+		this.spans[currentSpanIndex]++
+	}
+	this.lastUpdateTime = currentTime
+
+	for _, count := range this.spans {
+		result += count
+	}
+
+	return
+}
+
+func (this *Item[T]) Sum() (result T) {
+	if this.lastUpdateTime == 0 {
+		return 0
+	}
+
+	var currentTime = fasttime.Now().Unix()
+	var currentSpanIndex = this.calculateSpanIndex(currentTime)
+
+	if currentTime-this.lastUpdateTime > this.lifeSeconds {
+		return 0
+	} else {
+		var lastSpanIndex = this.calculateSpanIndex(this.lastUpdateTime)
+		var countSpans = len(this.spans)
+		for index := currentSpanIndex + 1; ; index++ {
+			var realIndex = index % countSpans
+			result += this.spans[realIndex]
+			if realIndex == lastSpanIndex {
+				break
+			}
+		}
+	}
+
 	return result
 }
 
-func (this *Item) Reset() {
-	for _, span := range this.spans {
-		span.Count = 0
-		span.Timestamp = 0
+func (this *Item[T]) Reset() {
+	for index := range this.spans {
+		this.spans[index] = 0
 	}
 }
 
-func (this *Item) IsExpired(currentTime int64) bool {
+func (this *Item[T]) IsExpired(currentTime int64) bool {
 	return this.lastUpdateTime < currentTime-this.lifeSeconds-this.spanSeconds
 }
+
+func (this *Item[T]) calculateSpanIndex(timestamp int64) int {
+	return int(timestamp % this.lifeSeconds / this.spanSeconds)
+}

internal/utils/counters/item_test.go

@@ -13,6 +13,27 @@ import (
 )
 
 func TestItem_Increase(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var item = counters.NewItem[uint32](10)
+	t.Log(item.Increase(), item.Sum())
+	time.Sleep(1 * time.Second)
+	t.Log(item.Increase(), item.Sum())
+	time.Sleep(2 * time.Second)
+	t.Log(item.Increase(), item.Sum())
+	time.Sleep(5 * time.Second)
+	t.Log(item.Increase(), item.Sum())
+	time.Sleep(6 * time.Second)
+	t.Log(item.Increase(), item.Sum())
+	time.Sleep(5 * time.Second)
+	t.Log(item.Increase(), item.Sum())
+	time.Sleep(11 * time.Second)
+	t.Log(item.Increase(), item.Sum())
+}
+
+func TestItem_Increase2(t *testing.T) {
 	// run only under single testing
 	if !testutils.IsSingleTesting() {
 		return
@@ -20,9 +41,9 @@ func TestItem_Increase(t *testing.T) {
 
 	var a = assert.NewAssertion(t)
 
-	var item = counters.NewItem(20)
+	var item = counters.NewItem[uint32](20)
 	for i := 0; i < 100; i++ {
-		t.Log(item.Increase(), timeutil.Format("i:s"))
+		t.Log(item.Increase(), item.Sum(), timeutil.Format("H:i:s"))
 		time.Sleep(2 * time.Second)
 	}
 
@@ -37,7 +58,7 @@ func TestItem_IsExpired(t *testing.T) {
 
 	var currentTime = time.Now().Unix()
 
-	var item = counters.NewItem(10)
+	var item = counters.NewItem[uint32](10)
 	t.Log(item.IsExpired(currentTime))
 	time.Sleep(10 * time.Second)
 	t.Log(item.IsExpired(currentTime))
@@ -48,9 +69,13 @@ func TestItem_IsExpired(t *testing.T) {
 func BenchmarkItem_Increase(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
-	var item = counters.NewItem(60)
+	b.ReportAllocs()
 
-	for i := 0; i < b.N; i++ {
-		item.Increase()
-	}
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var item = counters.NewItem[uint32](60)
+			item.Increase()
+			item.Sum()
+		}
+	})
 }

internal/utils/counters/span.go

@@ -1,12 +0,0 @@
-// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
-
-package counters
-
-type Span struct {
-	Timestamp int64
-	Count     uint64
-}
-
-func NewSpan() *Span {
-	return &Span{}
-}

internal/utils/dbs/db.go

@@ -12,11 +12,16 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fs"
 	_ "github.com/mattn/go-sqlite3"
+	"os"
 	"strings"
 	"sync"
 	"time"
 )
 
+const (
+	SyncMode = "OFF"
+)
+
 var errDBIsClosed = errors.New("the database is closed")
 
 type DB struct {
@@ -47,15 +52,17 @@ func open(dsn string, lock bool) (*DB, error) {
 		return nil, errors.New("can not open database when process is quiting")
 	}
 
+	// decode path
+	var path = dsn
+	var queryIndex = strings.Index(dsn, "?")
+	if queryIndex >= 0 {
+		path = path[:queryIndex]
+	}
+	path = strings.TrimSpace(strings.TrimPrefix(path, "file:"))
+
 	// locker
 	var locker *fsutils.Locker
 	if lock {
-		var path = dsn
-		var queryIndex = strings.Index(dsn, "?")
-		if queryIndex >= 0 {
-			path = path[:queryIndex]
-		}
-		path = strings.TrimSpace(strings.TrimPrefix(path, "file:"))
 		locker = fsutils.NewLocker(path)
 		err := locker.Lock()
 		if err != nil {
@@ -64,12 +71,30 @@ func open(dsn string, lock bool) (*DB, error) {
 		}
 	}
 
+	// check if closed successfully last time, if not we recover it
+	var walPath = path + "-wal"
+	_, statWalErr := os.Stat(walPath)
+	var shouldRecover = statWalErr == nil
+
 	// open
 	rawDB, err := sql.Open("sqlite3", dsn)
 	if err != nil {
 		return nil, err
 	}
 
+	if shouldRecover {
+		err = rawDB.Close()
+		if err != nil {
+			return nil, err
+		}
+
+		// open again
+		rawDB, err = sql.Open("sqlite3", dsn)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	var db = NewDB(rawDB, dsn)
 	db.locker = locker
 	return db, nil
@@ -178,6 +203,7 @@ func (this *DB) Close() error {
 	this.statusLocker.Unlock()
 
 	// waiting for updating operations to finish
+	var maxLoops = 5_000
 	for {
 		this.statusLocker.Lock()
 		var countUpdating = this.countUpdating
@@ -186,6 +212,11 @@ func (this *DB) Close() error {
 			break
 		}
 		time.Sleep(1 * time.Millisecond)
+
+		maxLoops--
+		if maxLoops <= 0 {
+			break
+		}
 	}
 
 	for _, batch := range this.batches {

internal/utils/dbs/db_test.go

@@ -3,12 +3,13 @@
 package dbs_test
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
 	"net/url"
 	"testing"
 )
 
 func TestParseDSN(t *testing.T) {
-	var dsn = "file:/home/cache/p43/.indexes/db-3.db?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=88000"
+	var dsn = "file:/home/cache/p43/.indexes/db-3.db?cache=private&mode=ro&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_cache_size=88000"
 	u, err := url.Parse(dsn)
 	if err != nil {
 		t.Fatal(err)

internal/utils/dbs/stmt.go

@@ -5,6 +5,7 @@ package dbs
 import (
 	"context"
 	"database/sql"
+	fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
 )
 
 type Stmt struct {
@@ -27,7 +28,7 @@ func (this *Stmt) EnableStat() {
 	this.enableStat = true
 }
 
-func (this *Stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) {
+func (this *Stmt) ExecContext(ctx context.Context, args ...any) (result sql.Result, err error) {
 	// check database status
 	if this.db.BeginUpdating() {
 		defer this.db.EndUpdating()
@@ -38,10 +39,13 @@ func (this *Stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, err
 	if this.enableStat {
 		defer SharedQueryStatManager.AddQuery(this.query).End()
 	}
-	return this.rawStmt.ExecContext(ctx, args...)
+	fsutils.WriteBegin()
+	result, err = this.rawStmt.ExecContext(ctx, args...)
+	fsutils.WriteEnd()
+	return
 }
 
-func (this *Stmt) Exec(args ...any) (sql.Result, error) {
+func (this *Stmt) Exec(args ...any) (result sql.Result, err error) {
 	// check database status
 	if this.db.BeginUpdating() {
 		defer this.db.EndUpdating()
@@ -52,7 +56,11 @@ func (this *Stmt) Exec(args ...any) (sql.Result, error) {
 	if this.enableStat {
 		defer SharedQueryStatManager.AddQuery(this.query).End()
 	}
-	return this.rawStmt.Exec(args...)
+
+	fsutils.WriteBegin()
+	result, err = this.rawStmt.Exec(args...)
+	fsutils.WriteEnd()
+	return
 }
 
 func (this *Stmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) {

internal/utils/expires/list_test.go

@@ -2,6 +2,7 @@ package expires
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/logs"
 	timeutil "github.com/iwind/TeaGo/utils/time"
@@ -84,6 +85,10 @@ func TestList_GC_Batch(t *testing.T) {
 }
 
 func TestList_Start_GC(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	list := NewList()
 	list.Add(1, time.Now().Unix()+1)
 	list.Add(2, time.Now().Unix()+1)
@@ -125,6 +130,25 @@ func TestList_ManyItems(t *testing.T) {
 	t.Log(time.Since(now))
 }
 
+func TestList_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var list = NewList()
+
+	testutils.StartMemoryStats(t, func() {
+		t.Log(list.Count(), "items")
+	})
+
+
+	for i := 0; i < 10_000_000; i++ {
+		list.Add(uint64(i), time.Now().Unix()+1800)
+	}
+
+	time.Sleep(1 * time.Hour)
+}
+
 func TestList_Map_Performance(t *testing.T) {
 	t.Log("max uint32", math.MaxUint32)
 

internal/utils/expires/manager.go

@@ -33,7 +33,7 @@ func (this *Manager) init() {
 	for range this.ticker.C {
 		var currentTime = time.Now().Unix()
 		if lastTimestamp == 0 {
-			lastTimestamp = currentTime - 3600
+			lastTimestamp = currentTime - 86400 // prevent timezone changes
 		}
 
 		if currentTime >= lastTimestamp {

internal/utils/fnv/hash_test.go

@@ -14,3 +14,11 @@ func TestHash(t *testing.T) {
 		t.Log(key + " => " + types.String(h))
 	}
 }
+
+func BenchmarkHashString(b *testing.B) {
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			fnv.HashString("abcdefh")
+		}
+	})
+}

internal/utils/fs/disk.go

@@ -4,11 +4,20 @@ package fsutils
 
 import (
 	"bytes"
+	"encoding/json"
+	"github.com/iwind/TeaGo/Tea"
 	"math"
 	"os"
 	"time"
 )
 
+const diskSpeedDataFile = "disk.speed.json"
+
+type DiskSpeedCache struct {
+	Speed   Speed   `json:"speed"`
+	SpeedMB float64 `json:"speedMB"`
+}
+
 // CheckDiskWritingSpeed test disk writing speed
 func CheckDiskWritingSpeed() (speedMB float64, err error) {
 	var tempDir = os.TempDir()
@@ -66,8 +75,13 @@ func CheckDiskIsFast() (speedMB float64, isFast bool, err error) {
 	if err != nil {
 		return
 	}
+
 	isFast = speedMB > 150
 
+	if speedMB <= DiskSpeedMB {
+		return
+	}
+
 	if speedMB > 1000 {
 		DiskSpeed = SpeedExtremelyFast
 	} else if speedMB > 150 {
@@ -79,8 +93,15 @@ func CheckDiskIsFast() (speedMB float64, isFast bool, err error) {
 	}
 	calculateDiskMaxWrites()
 
-	if speedMB > DiskSpeedMB {
-		DiskSpeedMB = speedMB
+	DiskSpeedMB = speedMB
+
+	// write to local file
+	cacheData, jsonErr := json.Marshal(&DiskSpeedCache{
+		Speed:   DiskSpeed,
+		SpeedMB: DiskSpeedMB,
+	})
+	if jsonErr == nil {
+		_ = os.WriteFile(Tea.Root+"/data/"+diskSpeedDataFile, cacheData, 0666)
 	}
 
 	return

internal/utils/fs/stat.go

@@ -8,8 +8,8 @@ import (
 	"sync"
 )
 
-// Stat device contains the path
-func Stat(path string) (*StatResult, error) {
+// StatDevice device contains the path
+func StatDevice(path string) (*StatResult, error) {
 	var stat = &unix.Statfs_t{}
 	err := unix.Statfs(path, stat)
 	if err != nil {
@@ -23,8 +23,8 @@ var cacheMap = map[string]*StatResult{} // path => StatResult
 
 const cacheLife = 3 // seconds
 
-// StatCache stat device with cache
-func StatCache(path string) (*StatResult, error) {
+// StatDeviceCache stat device with cache
+func StatDeviceCache(path string) (*StatResult, error) {
 	locker.RLock()
 	stat, ok := cacheMap[path]
 	if ok && stat.updatedAt >= fasttime.Now().Unix()-cacheLife {
@@ -36,7 +36,7 @@ func StatCache(path string) (*StatResult, error) {
 	locker.Lock()
 	defer locker.Unlock()
 
-	stat, err := Stat(path)
+	stat, err := StatDevice(path)
 	if err != nil {
 		return nil, err
 	}

internal/utils/fs/stat_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestStat(t *testing.T) {
-	stat, err := fsutils.Stat("/usr/local")
+	stat, err := fsutils.StatDevice("/usr/local")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -19,7 +19,7 @@ func TestStat(t *testing.T) {
 
 func TestStatCache(t *testing.T) {
 	for i := 0; i < 10; i++ {
-		stat, err := fsutils.StatCache("/usr/local")
+		stat, err := fsutils.StatDeviceCache("/usr/local")
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -40,16 +40,16 @@ func TestConcurrent(t *testing.T) {
 		go func() {
 			defer wg.Done()
 
-			_, _ = fsutils.Stat("/usr/local")
+			_, _ = fsutils.StatDevice("/usr/local")
 		}()
 	}
 	wg.Wait()
 }
 
-func BenchmarkStat(b *testing.B) {
+func BenchmarkStatDevice(b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			_, err := fsutils.Stat("/usr/local")
+			_, err := fsutils.StatDevice("/usr/local")
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -57,10 +57,10 @@ func BenchmarkStat(b *testing.B) {
 	})
 }
 
-func BenchmarkStatCache(b *testing.B) {
+func BenchmarkStatCacheDevice(b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			_, err := fsutils.StatCache("/usr/local")
+			_, err := fsutils.StatDeviceCache("/usr/local")
 			if err != nil {
 				b.Fatal(err)
 			}

internal/utils/fs/status.go

@@ -3,7 +3,11 @@
 package fsutils
 
 import (
+	"encoding/json"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/shirou/gopsutil/v3/load"
+	"os"
 	"sync/atomic"
 	"time"
 )
@@ -37,6 +41,14 @@ var (
 	DiskSpeedMB   float64
 )
 
+var IsInHighLoad = false
+var IsInExtremelyHighLoad = false
+
+const (
+	highLoad1Threshold          = 20
+	extremelyHighLoad1Threshold = 40
+)
+
 func init() {
 	if !teaconst.IsMain {
 		return
@@ -44,6 +56,18 @@ func init() {
 
 	// test disk
 	go func() {
+		// load last result from local disk
+		cacheData, cacheErr := os.ReadFile(Tea.Root + "/data/" + diskSpeedDataFile)
+		if cacheErr == nil {
+			var cache = &DiskSpeedCache{}
+			err := json.Unmarshal(cacheData, cache)
+			if err == nil && cache.SpeedMB > 0 {
+				DiskSpeedMB = cache.SpeedMB
+				DiskSpeed = cache.Speed
+				calculateDiskMaxWrites()
+			}
+		}
+
 		// initial check
 		_, _, _ = CheckDiskIsFast()
 
@@ -60,6 +84,16 @@ func init() {
 			}
 		}
 	}()
+
+	// check high load
+	go func() {
+		var ticker = time.NewTicker(5 * time.Second)
+		for range ticker.C {
+			stat, _ := load.Avg()
+			IsInExtremelyHighLoad = stat != nil && stat.Load1 > extremelyHighLoad1Threshold
+			IsInHighLoad = stat != nil && stat.Load1 > highLoad1Threshold && !DiskIsFast()
+		}
+	}()
 }
 
 func DiskIsFast() bool {
@@ -67,12 +101,20 @@ func DiskIsFast() bool {
 }
 
 func DiskIsExtremelyFast() bool {
+	// 在开发环境下返回false，以便于测试
+	if Tea.IsTesting() {
+		return false
+	}
 	return DiskSpeed == SpeedExtremelyFast
 }
 
 var countWrites int32 = 0
 
 func WriteReady() bool {
+	if IsInExtremelyHighLoad {
+		return false
+	}
+
 	return atomic.LoadInt32(&countWrites) < DiskMaxWrites
 }
 

internal/utils/ip.go

@@ -72,3 +72,18 @@ func IsIPv6(ip string) bool {
 	}
 	return !IsIPv4(ip)
 }
+
+// IsWildIP 宽泛地判断一个数据是否为IP
+func IsWildIP(v string) bool {
+	var l = len(v)
+	if l == 0 {
+		return false
+	}
+
+	// for [IPv6]
+	if v[0] == '[' && v[l-1] == ']' {
+		return IsWildIP(v[1 : l-1])
+	}
+
+	return net.ParseIP(v) != nil
+}

internal/utils/ip_test.go

@@ -1,51 +1,65 @@
-package utils
+package utils_test
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/assert"
 	"testing"
 )
 
 func TestIP2Long(t *testing.T) {
-	t.Log(IP2Long("0.0.0.0"))
-	t.Log(IP2Long("1.0.0.0"))
-	t.Log(IP2Long("0.0.0.0.0"))
-	t.Log(IP2Long("2001:db8:0:1::101"))
-	t.Log(IP2Long("2001:db8:0:1::102"))
-	t.Log(IP2Long("::1"))
+	t.Log(utils.IP2Long("0.0.0.0"))
+	t.Log(utils.IP2Long("1.0.0.0"))
+	t.Log(utils.IP2Long("0.0.0.0.0"))
+	t.Log(utils.IP2Long("2001:db8:0:1::101"))
+	t.Log(utils.IP2Long("2001:db8:0:1::102"))
+	t.Log(utils.IP2Long("::1"))
 }
 
 func TestIsLocalIP(t *testing.T) {
 	var a = assert.NewAssertion(t)
-	a.IsFalse(IsLocalIP("a"))
-	a.IsFalse(IsLocalIP("1.2.3"))
-	a.IsTrue(IsLocalIP("127.0.0.1"))
-	a.IsTrue(IsLocalIP("192.168.0.1"))
-	a.IsTrue(IsLocalIP("10.0.0.1"))
-	a.IsTrue(IsLocalIP("172.16.0.1"))
-	a.IsTrue(IsLocalIP("::1"))
-	a.IsFalse(IsLocalIP("::1:2:3"))
-	a.IsFalse(IsLocalIP("8.8.8.8"))
+	a.IsFalse(utils.IsLocalIP("a"))
+	a.IsFalse(utils.IsLocalIP("1.2.3"))
+	a.IsTrue(utils.IsLocalIP("127.0.0.1"))
+	a.IsTrue(utils.IsLocalIP("192.168.0.1"))
+	a.IsTrue(utils.IsLocalIP("10.0.0.1"))
+	a.IsTrue(utils.IsLocalIP("172.16.0.1"))
+	a.IsTrue(utils.IsLocalIP("::1"))
+	a.IsFalse(utils.IsLocalIP("::1:2:3"))
+	a.IsFalse(utils.IsLocalIP("8.8.8.8"))
 }
 
 func TestIsIPv4(t *testing.T) {
 	var a = assert.NewAssertion(t)
-	a.IsTrue(IsIPv4("192.168.1.1"))
-	a.IsTrue(IsIPv4("0.0.0.0"))
-	a.IsFalse(IsIPv4("192.168.1.256"))
-	a.IsFalse(IsIPv4("192.168.1"))
-	a.IsFalse(IsIPv4("::1"))
-	a.IsFalse(IsIPv4("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
-	a.IsFalse(IsIPv4("::ffff:192.168.0.1"))
+	a.IsTrue(utils.IsIPv4("192.168.1.1"))
+	a.IsTrue(utils.IsIPv4("0.0.0.0"))
+	a.IsFalse(utils.IsIPv4("192.168.1.256"))
+	a.IsFalse(utils.IsIPv4("192.168.1"))
+	a.IsFalse(utils.IsIPv4("::1"))
+	a.IsFalse(utils.IsIPv4("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
+	a.IsFalse(utils.IsIPv4("::ffff:192.168.0.1"))
 }
 
 func TestIsIPv6(t *testing.T) {
 	var a = assert.NewAssertion(t)
-	a.IsFalse(IsIPv6("192.168.1.1"))
-	a.IsFloat32(IsIPv6("0.0.0.0"))
-	a.IsFalse(IsIPv6("192.168.1.256"))
-	a.IsFalse(IsIPv6("192.168.1"))
-	a.IsTrue(IsIPv6("::1"))
-	a.IsTrue(IsIPv6("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
-	a.IsTrue(IsIPv4("::ffff:192.168.0.1"))
-	a.IsTrue(IsIPv6("::ffff:192.168.0.1"))
+	a.IsFalse(utils.IsIPv6("192.168.1.1"))
+	a.IsFloat32(utils.IsIPv6("0.0.0.0"))
+	a.IsFalse(utils.IsIPv6("192.168.1.256"))
+	a.IsFalse(utils.IsIPv6("192.168.1"))
+	a.IsTrue(utils.IsIPv6("::1"))
+	a.IsTrue(utils.IsIPv6("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
+	a.IsTrue(utils.IsIPv4("::ffff:192.168.0.1"))
+	a.IsTrue(utils.IsIPv6("::ffff:192.168.0.1"))
+}
+
+func TestIsWildIP(t *testing.T) {
+	var a = assert.NewAssertion(t)
+	a.IsTrue(utils.IsWildIP("192.168.1.100"))
+	a.IsTrue(utils.IsWildIP("::1"))
+	a.IsTrue(utils.IsWildIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
+	a.IsTrue(utils.IsWildIP("[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"))
+	a.IsFalse(utils.IsWildIP(""))
+	a.IsFalse(utils.IsWildIP("[]"))
+	a.IsFalse(utils.IsWildIP("[1]"))
+	a.IsFalse(utils.IsWildIP("192.168.2.256"))
+	a.IsFalse(utils.IsWildIP("192.168.2"))
 }

internal/utils/linkedlist/item.go

@@ -2,13 +2,13 @@
 
 package linkedlist
 
-type Item struct {
-	prev *Item
-	next *Item
+type Item[T any] struct {
+	prev *Item[T]
+	next *Item[T]
 
-	Value interface{}
+	Value T
 }
 
-func NewItem(value interface{}) *Item {
-	return &Item{Value: value}
+func NewItem[T any](value T) *Item[T] {
+	return &Item[T]{Value: value}
 }

internal/utils/linkedlist/list.go

@@ -2,25 +2,25 @@
 
 package linkedlist
 
-type List struct {
-	head  *Item
-	end   *Item
+type List[T any]  struct {
+	head  *Item[T]
+	end   *Item[T]
 	count int
 }
 
-func NewList() *List {
-	return &List{}
+func NewList[T any]() *List[T] {
+	return &List[T]{}
 }
 
-func (this *List) Head() *Item {
+func (this *List[T]) Head() *Item[T] {
 	return this.head
 }
 
-func (this *List) End() *Item {
+func (this *List[T]) End() *Item[T] {
 	return this.end
 }
 
-func (this *List) Push(item *Item) {
+func (this *List[T]) Push(item *Item[T]) {
 	if item == nil {
 		return
 	}
@@ -36,7 +36,7 @@ func (this *List) Push(item *Item) {
 	this.add(item)
 }
 
-func (this *List) Remove(item *Item) {
+func (this *List[T]) Remove(item *Item[T]) {
 	if item == nil {
 		return
 	}
@@ -58,11 +58,11 @@ func (this *List) Remove(item *Item) {
 	this.count--
 }
 
-func (this *List) Len() int {
+func (this *List[T]) Len() int {
 	return this.count
 }
 
-func (this *List) Range(f func(item *Item) (goNext bool)) {
+func (this *List[T]) Range(f func(item *Item[T]) (goNext bool)) {
 	for e := this.head; e != nil; e = e.next {
 		goNext := f(e)
 		if !goNext {
@@ -71,12 +71,12 @@ func (this *List) Range(f func(item *Item) (goNext bool)) {
 	}
 }
 
-func (this *List) Reset() {
+func (this *List[T]) Reset() {
 	this.head = nil
 	this.end = nil
 }
 
-func (this *List) add(item *Item) {
+func (this *List[T]) add(item *Item[T]) {
 	if item == nil {
 		return
 	}

internal/utils/linkedlist/list_test.go

@@ -5,6 +5,7 @@ package linkedlist_test
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
 	"runtime"
+	"strconv"
 	"testing"
 )
 
@@ -12,27 +13,48 @@ func TestNewList_Memory(t *testing.T) {
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 
-	var list = linkedlist.NewList()
+	var list = linkedlist.NewList[int]()
 	for i := 0; i < 1_000_000; i++ {
-		var item = &linkedlist.Item{}
+		var item = &linkedlist.Item[int]{}
 		list.Push(item)
 	}
 
+	runtime.GC()
+
 	var stat2 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat2)
-	t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
+	t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
 	t.Log(list.Len())
 
 	var count = 0
-	list.Range(func(item *linkedlist.Item) (goNext bool) {
+	list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
 		count++
 		return true
 	})
 	t.Log(count)
 }
 
+func TestNewList_Memory_String(t *testing.T) {
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
+	var list = linkedlist.NewList[string]()
+	for i := 0; i < 1_000_000; i++ {
+		var item = &linkedlist.Item[string]{}
+		item.Value = strconv.Itoa(i)
+		list.Push(item)
+	}
+
+	runtime.GC()
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+	t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
+	t.Log(list.Len())
+}
+
 func TestList_Push(t *testing.T) {
-	var list = linkedlist.NewList()
+	var list = linkedlist.NewList[int]()
 	list.Push(linkedlist.NewItem(1))
 	list.Push(linkedlist.NewItem(2))
 
@@ -41,42 +63,42 @@ func TestList_Push(t *testing.T) {
 
 	var item4 = linkedlist.NewItem(4)
 	list.Push(item4)
-	list.Range(func(item *linkedlist.Item) (goNext bool) {
+	list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
 		t.Log(item.Value)
 		return true
 	})
 
-	t.Log("=== after push3 ===")
+	t.Log("=== after push 3 ===")
 	list.Push(item3)
-	list.Range(func(item *linkedlist.Item) (goNext bool) {
+	list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
 		t.Log(item.Value)
 		return true
 	})
 
-	t.Log("=== after push4 ===")
+	t.Log("=== after push 4 ===")
 	list.Push(item4)
 	list.Push(item3)
 	list.Push(item3)
 	list.Push(item3)
 	list.Push(item4)
 	list.Push(item4)
-	list.Range(func(item *linkedlist.Item) (goNext bool) {
+	list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
 		t.Log(item.Value)
 		return true
 	})
 
-	t.Log("=== after remove ===")
+	t.Log("=== after remove 3 ===")
 	list.Remove(item3)
-	list.Range(func(item *linkedlist.Item) (goNext bool) {
+	list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
 		t.Log(item.Value)
 		return true
 	})
 }
 
 func BenchmarkList_Add(b *testing.B) {
-	var list = linkedlist.NewList()
+	var list = linkedlist.NewList[int]()
 	for i := 0; i < b.N; i++ {
-		var item = &linkedlist.Item{}
+		var item = &linkedlist.Item[int]{}
 		list.Push(item)
 	}
 }

internal/utils/string.go

@@ -56,3 +56,16 @@ func EqualStrings(s1 []string, s2 []string) bool {
 	}
 	return true
 }
+
+// CutPrefix returns s without the provided leading prefix string
+// and reports whether it found the prefix.
+// If s doesn't start with prefix, CutPrefix returns s, false.
+// If prefix is the empty string, CutPrefix returns s, true.
+//
+// copy from go source
+func CutPrefix(s, prefix string) (after string, found bool) {
+	if !strings.HasPrefix(s, prefix) {
+		return s, false
+	}
+	return s[len(prefix):], true
+}