版本号修改为1.3.3

build/build.sh

@@ -6,10 +6,11 @@ function build() {
 	VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
 	DIST=$ROOT/"../dist/${NAME}"
 	MUSL_DIR="/usr/local/opt/musl-cross/bin"
+	SRCDIR=$(realpath "$ROOT/..")
 
 	# for macOS users: precompiled gcc can be downloaded from https://github.com/messense/homebrew-macos-cross-toolchains
 	GCC_X86_64_DIR="/usr/local/gcc/x86_64-unknown-linux-gnu/bin"
-	GCC_ARM64_DIR="//usr/local/gcc/aarch64-unknown-linux-gnu/bin"
+	GCC_ARM64_DIR="/usr/local/gcc/aarch64-unknown-linux-gnu/bin"
 
 	OS=${1}
 	ARCH=${2}
@@ -70,6 +71,8 @@ function build() {
 
 	CC_PATH=""
 	CXX_PATH=""
+	CGO_LDFLAGS=""
+	CGO_CFLAGS=""
 	BUILD_TAG=$TAG
 	if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
 		if [ "${ARCH}" == "amd64" ]; then
@@ -79,7 +82,7 @@ function build() {
 				CC_PATH="x86_64-unknown-linux-gnu-gcc"
 				CXX_PATH="x86_64-unknown-linux-gnu-g++"
 				if [ "$TAG" = "plus" ]; then
-					BUILD_TAG="plus,script"
+					BUILD_TAG="plus,script,packet"
 				fi
 			else
 				CC_PATH="x86_64-linux-musl-gcc"
@@ -97,7 +100,7 @@ function build() {
 				CC_PATH="aarch64-unknown-linux-gnu-gcc"
 				CXX_PATH="aarch64-unknown-linux-gnu-g++"
 				if [ "$TAG" = "plus" ]; then
-					BUILD_TAG="plus,script"
+					BUILD_TAG="plus,script,packet"
 				fi
 			else
 				CC_PATH="aarch64-linux-musl-gcc"
@@ -117,13 +120,26 @@ function build() {
 			CXX_PATH="mips64el-linux-musl-g++"
 		fi
 	fi
+
+	# libpcap
+	if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] &&  [ "$TAG" == "plus" ]; then
+		CGO_LDFLAGS="-L${SRCDIR}/libs/libpcap/${ARCH} -lpcap -L${SRCDIR}/libs/libbrotli/${ARCH} -lbrotlienc -lbrotlidec -lbrotlicommon"
+		CGO_CFLAGS="-I${SRCDIR}/libs/libpcap/src/libpcap -I${SRCDIR}/libs/libpcap/src/libpcap/pcap -I${SRCDIR}/libs/libbrotli/src/brotli/c/include"
+	fi
+
 	if [ ! -z $CC_PATH ]; then
-		env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
+		env CC=$MUSL_DIR/$CC_PATH \
+		 CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" \
+		 GOARCH="${ARCH}" CGO_ENABLED=1 \
+		 CGO_LDFLAGS="${CGO_LDFLAGS}" \
+		 CGO_CFLAGS="${CGO_CFLAGS}" \
+		 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
 	else
 		if [[ `uname` == *"Linux"* ]] && [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] &&  [ "$TAG" == "plus" ]; then
-			BUILD_TAG="plus,script"
+			BUILD_TAG="plus,script,packet"
 		fi
-		env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
+
+		env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1  CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
 	fi
 
 	if [ ! -f "${DIST}/bin/${NAME}" ]; then

build/test.sh

@@ -6,4 +6,6 @@ if [ -z "$TAG" ]; then
 	TAG="community"
 fi
 
-go test -v ../... -tags=${TAG}
+# reference: https://pkg.go.dev/cmd/go/internal/test
+go clean -testcache
+go test -timeout 10s -tags="${TAG}" -cover ../...

cmd/edge-node/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/types"
 	"github.com/iwind/gosock/pkg/gosock"
+	"gopkg.in/yaml.v3"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -30,7 +31,7 @@ func main() {
 	var app = apps.NewAppCmd().
 		Version(teaconst.Version).
 		Product(teaconst.ProductName).
-		Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog|uninstall]").
+		Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|config|pprof|accesslog|uninstall]").
 		Usage(teaconst.ProcessName + " [trackers|goman|conns|gc|bandwidth|disk|cache.garbage]").
 		Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove|ip.close] IP")
 
@@ -228,11 +229,18 @@ func main() {
 	})
 	app.On("gc", func() {
 		var sock = gosock.NewTmpSock(teaconst.ProcessName)
-		_, err := sock.Send(&gosock.Command{Code: "gc"})
+		reply, err := sock.Send(&gosock.Command{Code: "gc"})
 		if err != nil {
 			fmt.Println("[ERROR]" + err.Error())
 		} else {
-			fmt.Println("ok")
+			if reply == nil {
+				fmt.Println("ok")
+			} else {
+				var paramMap = maps.NewMap(reply.Params)
+				var pauseMS = paramMap.GetFloat64("pauseMS")
+				var costMS = paramMap.GetFloat64("costMS")
+				fmt.Printf("ok, cost: %.4fms, pause: %.4fms", costMS, pauseMS)
+			}
 		}
 	})
 	app.On("ip.drop", func() {
@@ -517,6 +525,41 @@ func main() {
 			fmt.Println("[ERROR]" + params.GetString("error"))
 		}
 	})
+	app.On("config", func() {
+		var configString = os.Args[len(os.Args)-1]
+		if configString == "config" {
+			fmt.Println("Usage: edge-node config '\nrpc.endpoints: [\"...\"]\nnodeId: \"...\"\nsecret: \"...\"\n'")
+			return
+		}
+
+		var config = &configs.APIConfig{}
+		err := yaml.Unmarshal([]byte(configString), config)
+		if err != nil {
+			fmt.Println("[ERROR]decode config failed: " + err.Error())
+			return
+		}
+
+		err = config.Init()
+		if err != nil {
+			fmt.Println("[ERROR]validate config failed: " + err.Error())
+			return
+		}
+
+		// marshal again
+		configYAML, err := yaml.Marshal(config)
+		if err != nil {
+			fmt.Println("[ERROR]encode config failed: " + err.Error())
+			return
+		}
+
+		err = os.WriteFile(Tea.Root + "/configs/api_node.yaml", configYAML, 0666)
+		if err != nil {
+			fmt.Println("[ERROR]write config failed: " + err.Error())
+			return
+		}
+
+		fmt.Println("success")
+	})
 	app.Run(func() {
 		var node = nodes.NewNode()
 		node.Start()

go.mod

@@ -25,28 +25,27 @@ require (
 	github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d
 	github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
 	github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
-	github.com/iwind/gowebp v0.0.0-20230927084601-21954d2e229f
-	github.com/klauspost/compress v1.16.5
+	github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4
+	github.com/klauspost/compress v1.17.2
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mdlayher/netlink v1.7.1
 	github.com/miekg/dns v1.1.43
 	github.com/mssola/useragent v1.0.0
 	github.com/pires/go-proxyproto v0.6.1
 	github.com/qiniu/go-sdk/v7 v7.16.0
-	github.com/quic-go/quic-go v0.39.0
+	github.com/quic-go/quic-go v0.39.2
 	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.41
-	golang.org/x/image v0.7.0
+	golang.org/x/image v0.13.0
 	golang.org/x/net v0.17.0
 	golang.org/x/sys v0.13.0
-	google.golang.org/grpc v1.55.0
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/grpc v1.59.0
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chai2010/webp v1.1.1 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -54,7 +53,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
-	github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 // indirect
+	github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.0.0 // indirect
 	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
@@ -62,10 +61,10 @@ require (
 	github.com/mdlayher/socket v0.4.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/mozillazg/go-httpheader v0.2.1 // indirect
-	github.com/onsi/ginkgo/v2 v2.12.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.13.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.3.4 // indirect
+	github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
 	github.com/tdewolff/minify/v2 v2.12.7 // indirect
 	github.com/tdewolff/parse/v2 v2.6.6 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
@@ -73,11 +72,11 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.uber.org/mock v0.3.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 )

go.sum

@@ -15,8 +15,6 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
-github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
 github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
 github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
@@ -55,10 +53,10 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8 h1:gpptm606MZYGaMHMsB4Srmb6EbW/IVHnt04rcMXnkBQ=
-github.com/google/pprof v0.0.0-20230912144702-c363fe2c2ed8/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0=
+github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible h1:XRAk4HBDLCYEdPLWtKf5iZhOi7lfx17aY0oSO9+mcg8=
 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
 github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d h1:XnTIj781NdSipts60fVqbgZorVAKVSRaA6nqVNfBQ1g=
@@ -69,8 +67,8 @@ github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedV
 github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
 github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
 github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
-github.com/iwind/gowebp v0.0.0-20230927084601-21954d2e229f h1:DCUsOhpZbuKiROTZGc9V9z1uEfm+EbU5nhze+Tv5xo0=
-github.com/iwind/gowebp v0.0.0-20230927084601-21954d2e229f/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
+github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4 h1:eyymORsZg0tZ0niyolYF4nao4sdNUI+Ll40s96tKHBY=
+github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
 github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4 h1:RPAH9Sj9l/20zH5zU5/iJGszfwPq6eLjoiC/n/asulA=
 github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4/go.mod h1:7OLL+86wZKfBnAJxNxmdcZ0ebbgdp/A28fcagx9oJqA=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
@@ -81,8 +79,8 @@ github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTx
 github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
 github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
-github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
-github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -111,8 +109,8 @@ github.com/mssola/useragent v1.0.0 h1:WRlDpXyxHDNfvZaPEut5Biveq86Ze4o4EMffyMxmH5
 github.com/mssola/useragent v1.0.0/go.mod h1:hz9Cqz4RXusgg1EdI4Al0INR62kP7aPSRNHnpU+b85Y=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA=
-github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
 github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
@@ -128,10 +126,10 @@ github.com/qiniu/go-sdk/v7 v7.16.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYX
 github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
 github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
-github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg=
-github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.39.0 h1:AgP40iThFMY0bj8jGxROhw3S0FMGa8ryqsmi9tBH3So=
-github.com/quic-go/quic-go v0.39.0/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
+github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
+github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
+github.com/quic-go/quic-go v0.39.2 h1:hmwAf8zAHlvan0Y5PXxeeBFZEW17IW99sXLry8I2kjk=
+github.com/quic-go/quic-go v0.39.2/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
@@ -171,36 +169,29 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
-golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
-golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
+golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -215,24 +206,18 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -241,20 +226,19 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
-google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

internal/caches/errors.go

@@ -34,16 +34,14 @@ func CanIgnoreErr(err error) bool {
 	if err == nil {
 		return true
 	}
-	if err == ErrFileIsWriting ||
-		err == ErrEntityTooLarge ||
-		err == ErrWritingUnavailable ||
-		err == ErrWritingQueueFull ||
-		err == ErrServerIsBusy {
+	if errors.Is(err, ErrFileIsWriting) ||
+		errors.Is(err, ErrEntityTooLarge) ||
+		errors.Is(err, ErrWritingUnavailable) ||
+		errors.Is(err, ErrWritingQueueFull) ||
+		errors.Is(err, ErrServerIsBusy) {
 		return true
 	}
-	_, ok := err.(*CapacityError)
-	if ok {
-		return true
-	}
-	return false
+
+	var capacityErr *CapacityError
+	return errors.As(err, &capacityErr)
 }

internal/caches/errros_test.go

@@ -1,16 +1,24 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package caches
+package caches_test
 
 import (
+	"errors"
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/iwind/TeaGo/assert"
 	"testing"
 )
 
 func TestCanIgnoreErr(t *testing.T) {
-	a := assert.NewAssertion(t)
+	var a = assert.NewAssertion(t)
 
-	a.IsTrue(CanIgnoreErr(ErrFileIsWriting))
-	a.IsTrue(CanIgnoreErr(NewCapacityError("over capcity")))
-	a.IsFalse(CanIgnoreErr(ErrNotFound))
+	a.IsTrue(caches.CanIgnoreErr(caches.ErrFileIsWriting))
+	a.IsTrue(caches.CanIgnoreErr(fmt.Errorf("error: %w", caches.ErrFileIsWriting)))
+	a.IsTrue(errors.Is(fmt.Errorf("error: %w", caches.ErrFileIsWriting), caches.ErrFileIsWriting))
+	a.IsTrue(errors.Is(caches.ErrFileIsWriting, caches.ErrFileIsWriting))
+	a.IsTrue(caches.CanIgnoreErr(caches.NewCapacityError("over capacity")))
+	a.IsTrue(caches.CanIgnoreErr(fmt.Errorf("error: %w", caches.NewCapacityError("over capacity"))))
+	a.IsFalse(caches.CanIgnoreErr(caches.ErrNotFound))
+	a.IsFalse(caches.CanIgnoreErr(errors.New("test error")))
 }

internal/caches/item_test.go

@@ -4,6 +4,7 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
@@ -18,7 +19,11 @@ func TestItems_Memory(t *testing.T) {
 	var memory1 = stat.HeapInuse
 
 	var items = []*caches.Item{}
-	for i := 0; i < 10_000_000; i++ {
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 10_000_000
+	}
+	for i := 0; i < count; i++ {
 		items = append(items, &caches.Item{
 			Key: types.String(i),
 		})
@@ -33,7 +38,9 @@ func TestItems_Memory(t *testing.T) {
 	var memory3 = stat.HeapInuse
 	t.Log(memory2, memory3, (memory3-memory2)/1024/1024, "M")
 
-	time.Sleep(1 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(1 * time.Second)
+	}
 }
 
 func TestItems_Memory2(t *testing.T) {
@@ -42,7 +49,12 @@ func TestItems_Memory2(t *testing.T) {
 	var memory1 = stat.HeapInuse
 
 	var items = map[int32]map[string]zero.Zero{}
-	for i := 0; i < 10_000_000; i++ {
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 10_000_000
+	}
+
+	for i := 0; i < count; i++ {
 		var week = int32((time.Now().Unix() - int64(86400*rands.Int(0, 300))) / (86400 * 7))
 		m, ok := items[week]
 		if !ok {
@@ -57,7 +69,9 @@ func TestItems_Memory2(t *testing.T) {
 
 	t.Log(memory1, memory2, (memory2-memory1)/1024/1024, "M")
 
-	time.Sleep(1 * time.Second)
+	if testutils.IsSingleTesting() {
+		time.Sleep(1 * time.Second)
+	}
 	for w, i := range items {
 		t.Log(w, len(i))
 	}

internal/caches/list_file.go

@@ -566,7 +566,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
 }
 
 func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
-	var maxTimestamp = fasttime.Now().Unix() + 7200
+	var maxTimestamp = fasttime.Now().Unix() + 3600
 	if expiresAt > maxTimestamp {
 		return maxTimestamp
 	}

internal/caches/list_file_db.go

@@ -12,7 +12,6 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/types"
-	timeutil "github.com/iwind/TeaGo/utils/time"
 	"net"
 	"net/url"
 	"os"
@@ -48,11 +47,10 @@ type FileListDB struct {
 	deleteByHashStmt *dbs.Stmt // 根据hash删除数据
 	deleteByHashSQL  string
 
-	statStmt             *dbs.Stmt // 统计
-	purgeStmt            *dbs.Stmt // 清理
-	deleteAllStmt        *dbs.Stmt // 删除所有数据
-	listOlderItemsStmt   *dbs.Stmt // 读取较早存储的缓存
-	updateAccessWeekStmt *dbs.Stmt // 修改访问日期
+	statStmt           *dbs.Stmt // 统计
+	purgeStmt          *dbs.Stmt // 清理
+	deleteAllStmt      *dbs.Stmt // 删除所有数据
+	listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
 }
 
 func NewFileListDB() *FileListDB {
@@ -65,10 +63,10 @@ func (this *FileListDB) Open(dbPath string) error {
 	this.dbPath = dbPath
 
 	// 动态调整Cache值
-	var cacheSize = 32000
+	var cacheSize = 512
 	var memoryGB = utils.SystemMemoryGB()
-	if memoryGB >= 8 {
-		cacheSize += 32000 * memoryGB / 8
+	if memoryGB >= 1 {
+		cacheSize = 256 * memoryGB
 	}
 
 	// write db
@@ -136,7 +134,7 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt", "accessWeek") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
 	if err != nil {
 		return err
@@ -173,12 +171,7 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.updateAccessWeekStmt, err = this.writeDB.Prepare(`UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`)
-	if err != nil {
-		return err
-	}
-
-	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
+	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
 	if err != nil {
 		return err
 	}
@@ -213,7 +206,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
 		item.StaleAt = item.ExpiredAt
 	}
 
-	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix(), timeutil.Format("YW"))
+	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
 	if err != nil {
 		return this.WrapError(err)
 	}
@@ -309,8 +302,8 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
 }
 
 func (this *FileListDB) IncreaseHitAsync(hash string) error {
-	_, err := this.updateAccessWeekStmt.Exec(timeutil.Format("YW"), hash)
-	return err
+	// do nothing
+	return nil
 }
 
 func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -458,9 +451,6 @@ func (this *FileListDB) Close() error {
 	if this.deleteAllStmt != nil {
 		_ = this.deleteAllStmt.Close()
 	}
-	if this.updateAccessWeekStmt != nil {
-		_ = this.updateAccessWeekStmt.Close()
-	}
 	if this.listOlderItemsStmt != nil {
 		_ = this.listOlderItemsStmt.Close()
 	}
@@ -516,8 +506,7 @@ func (this *FileListDB) initTables(times int) error {
   "staleAt" integer DEFAULT 0,
   "createdAt" integer DEFAULT 0,
   "host" varchar(128),
-  "serverId" integer,
-  "accessWeek" varchar(6)
+  "serverId" integer
 );
 
 DROP INDEX IF EXISTS "createdAt";
@@ -533,8 +522,6 @@ CREATE INDEX IF NOT EXISTS "hash"
 ON "` + this.itemsTableName + `" (
   "hash" ASC
 );
-
-ALTER TABLE "cacheItems" ADD "accessWeek" varchar(6);
 `)
 
 		if err != nil {

internal/caches/list_file_db_test.go

@@ -4,12 +4,20 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
+	"runtime"
+	"runtime/debug"
 	"testing"
+	"time"
 )
 
 func TestFileListDB_ListLFUItems(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = caches.NewFileListDB()
 
 	defer func() {
@@ -34,6 +42,10 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
 }
 
 func TestFileListDB_CleanMatchKey(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = caches.NewFileListDB()
 
 	defer func() {
@@ -62,6 +74,10 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
 }
 
 func TestFileListDB_CleanMatchPrefix(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = caches.NewFileListDB()
 
 	defer func() {
@@ -88,3 +104,67 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestFileListDB_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var db = caches.NewFileListDB()
+
+	defer func() {
+		_ = db.Close()
+	}()
+
+	err := db.Open(Tea.Root + "/data/cache-index/p1/db-0.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log(db.Total())
+
+	// load hashes
+	var maxId int64
+	var hashList []string
+	var before = time.Now()
+	for i := 0; i < 1_000; i++ {
+		hashList, maxId, err = db.ListHashes(maxId)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(hashList) == 0 {
+			t.Log("hashes loaded", time.Since(before).Seconds()*1000, "ms")
+			break
+		}
+		if i%100 == 0 {
+			t.Log(i)
+		}
+	}
+
+	runtime.GC()
+	debug.FreeOSMemory()
+
+	//time.Sleep(600 * time.Second)
+
+	for i := 0; i < 1_000; i++ {
+		_, err = db.ListLFUItems(5000)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i%100 == 0 {
+			t.Log(i)
+		}
+	}
+
+	t.Log("loaded")
+
+	runtime.GC()
+	debug.FreeOSMemory()
+
+	time.Sleep(600 * time.Second)
+}

internal/caches/list_file_hash_map.go

@@ -9,11 +9,13 @@ import (
 	"sync"
 )
 
-const HashMapSharding = 11
+const HashMapSharding = 31
 
-var bigIntPool = sync.Pool{New: func() any {
-	return big.NewInt(0)
-}}
+var bigIntPool = sync.Pool{
+	New: func() any {
+		return big.NewInt(0)
+	},
+}
 
 // FileListHashMap 文件Hash列表
 type FileListHashMap struct {
@@ -63,7 +65,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
 		this.AddHashes(hashList)
 		lastId = maxId
 
-		maxLoops --
+		maxLoops--
 		if maxLoops <= 0 {
 			break
 		}
@@ -131,7 +133,10 @@ func (this *FileListHashMap) Clean() {
 		this.lockers[i].Lock()
 	}
 
-	this.m = make([]map[uint64]zero.Zero, HashMapSharding)
+	// 这里不能简单清空 this.m ，避免导致别的数据无法写入 map 而产生 panic
+	for i := 0; i < HashMapSharding; i++ {
+		this.m[i] = map[uint64]zero.Zero{}
+	}
 
 	for i := HashMapSharding - 1; i >= 0; i-- {
 		this.lockers[i].Unlock()

internal/caches/list_file_hash_map_test.go

@@ -4,8 +4,10 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
 	stringutil "github.com/iwind/TeaGo/utils/string"
@@ -85,6 +87,10 @@ func TestFileListHashMap_BigInt(t *testing.T) {
 }
 
 func TestFileListHashMap_Load(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
 
 	defer func() {
@@ -112,6 +118,25 @@ func TestFileListHashMap_Load(t *testing.T) {
 	}
 }
 
+func TestFileListHashMap_Delete(t *testing.T) {
+	var a = assert.NewAssertion(t)
+
+	var m = caches.NewFileListHashMap()
+	m.SetIsReady(true)
+	m.SetIsAvailable(true)
+	m.Add("a")
+	a.IsTrue(m.Len() == 1)
+	m.Delete("a")
+	a.IsTrue(m.Len() == 0)
+}
+
+func TestFileListHashMap_Clean(t *testing.T) {
+	var m = caches.NewFileListHashMap()
+	m.SetIsAvailable(true)
+	m.Clean()
+	m.Add("a")
+}
+
 func Benchmark_BigInt(b *testing.B) {
 	var hash = stringutil.Md5("123456")
 	b.ResetTimer()

internal/caches/list_file_test.go

@@ -17,6 +17,10 @@ import (
 )
 
 func TestFileList_Init(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
@@ -34,6 +38,10 @@ func TestFileList_Init(t *testing.T) {
 }
 
 func TestFileList_Add(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
 
 	defer func() {
@@ -107,6 +115,10 @@ func TestFileList_Add_Many(t *testing.T) {
 }
 
 func TestFileList_Exist(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
 	defer func() {
 		_ = list.Close()
@@ -143,6 +155,10 @@ func TestFileList_Exist(t *testing.T) {
 }
 
 func TestFileList_Exist_Many_DB(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	// 测试在多个数据库下的性能
 	var listSlice = []caches.ListInterface{}
 	for i := 1; i <= 10; i++ {
@@ -202,6 +218,10 @@ func TestFileList_Exist_Many_DB(t *testing.T) {
 }
 
 func TestFileList_CleanPrefix(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
@@ -222,6 +242,10 @@ func TestFileList_CleanPrefix(t *testing.T) {
 }
 
 func TestFileList_Remove(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
 	defer func() {
 		_ = list.Close()
@@ -246,6 +270,10 @@ func TestFileList_Remove(t *testing.T) {
 }
 
 func TestFileList_Purge(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
@@ -270,6 +298,10 @@ func TestFileList_Purge(t *testing.T) {
 }
 
 func TestFileList_PurgeLFU(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
@@ -294,6 +326,10 @@ func TestFileList_PurgeLFU(t *testing.T) {
 }
 
 func TestFileList_Stat(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {
@@ -313,6 +349,10 @@ func TestFileList_Stat(t *testing.T) {
 }
 
 func TestFileList_Count(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data")
 
 	defer func() {
@@ -333,6 +373,10 @@ func TestFileList_Count(t *testing.T) {
 }
 
 func TestFileList_CleanAll(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data")
 
 	defer func() {
@@ -352,6 +396,10 @@ func TestFileList_CleanAll(t *testing.T) {
 }
 
 func TestFileList_UpgradeV3(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList)
 
 	defer func() {
@@ -376,6 +424,10 @@ func TestFileList_UpgradeV3(t *testing.T) {
 }
 
 func BenchmarkFileList_Exist(b *testing.B) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 
 	defer func() {

internal/caches/list_memory.go

@@ -400,7 +400,19 @@ func (this *MemoryList) IncreaseHit(hash string) error {
 	return nil
 }
 
-func (this *MemoryList) print(t *testing.T) {
+func (this *MemoryList) Prefixes() []string {
+	return this.prefixes
+}
+
+func (this *MemoryList) ItemMaps() map[string]map[string]*Item {
+	return this.itemMaps
+}
+
+func (this *MemoryList) PurgeIndex() int {
+	return this.purgeIndex
+}
+
+func (this *MemoryList) Print(t *testing.T) {
 	this.locker.Lock()
 	for _, itemMap := range this.itemMaps {
 		if len(itemMap) > 0 {

internal/caches/list_memory_test.go

@@ -1,12 +1,14 @@
-package caches
+package caches_test
 
 import (
 	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/cespare/xxhash"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
+	stringutil "github.com/iwind/TeaGo/utils/string"
 	"math/rand"
 	"sort"
 	"strconv"
@@ -15,65 +17,65 @@ import (
 )
 
 func TestMemoryList_Add(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("123456", &Item{
+	_ = list.Add("123456", &caches.Item{
 		Key:        "c1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	t.Log(list.prefixes)
-	logs.PrintAsJSON(list.itemMaps, t)
+	t.Log(list.Prefixes())
+	logs.PrintAsJSON(list.ItemMaps(), t)
 	t.Log(list.Count())
 }
 
 func TestMemoryList_Remove(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
 	_ = list.Remove("b")
-	list.print(t)
+	list.Print(t)
 	t.Log(list.Count())
 }
 
 func TestMemoryList_Purge(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("c", &Item{
+	_ = list.Add("c", &caches.Item{
 		Key:        "c1",
 		ExpiredAt:  time.Now().Unix() - 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("d", &Item{
+	_ = list.Add("d", &caches.Item{
 		Key:        "d1",
 		ExpiredAt:  time.Now().Unix() - 2,
 		HeaderSize: 1024,
@@ -82,25 +84,30 @@ func TestMemoryList_Purge(t *testing.T) {
 		t.Log("delete:", hash)
 		return nil
 	})
-	list.print(t)
+	list.Print(t)
 
 	for i := 0; i < 1000; i++ {
 		_, _ = list.Purge(100, func(hash string) error {
 			t.Log("delete:", hash)
 			return nil
 		})
-		t.Log(list.purgeIndex)
+		t.Log(list.PurgeIndex())
 	}
 
 	t.Log(list.Count())
 }
 
 func TestMemoryList_Purge_Large_List(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	var list = caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
 
-	for i := 0; i < 1_000_000; i++ {
-		_ = list.Add("a"+strconv.Itoa(i), &Item{
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 1_000_000
+	}
+
+	for i := 0; i < count; i++ {
+		_ = list.Add("a"+strconv.Itoa(i), &caches.Item{
 			Key:        "a" + strconv.Itoa(i),
 			ExpiredAt:  time.Now().Unix() + int64(rands.Int(0, 24*3600)),
 			HeaderSize: 1024,
@@ -113,43 +120,46 @@ func TestMemoryList_Purge_Large_List(t *testing.T) {
 }
 
 func TestMemoryList_Stat(t *testing.T) {
-	list := NewMemoryList()
+	list := caches.NewMemoryList()
 	_ = list.Init()
-	_ = list.Add("a", &Item{
+	_ = list.Add("a", &caches.Item{
 		Key:        "a1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("b", &Item{
+	_ = list.Add("b", &caches.Item{
 		Key:        "b1",
 		ExpiredAt:  time.Now().Unix() + 3600,
 		HeaderSize: 1024,
 	})
-	_ = list.Add("c", &Item{
+	_ = list.Add("c", &caches.Item{
 		Key:        "c1",
 		ExpiredAt:  time.Now().Unix(),
 		HeaderSize: 1024,
 	})
-	_ = list.Add("d", &Item{
+	_ = list.Add("d", &caches.Item{
 		Key:        "d1",
 		ExpiredAt:  time.Now().Unix() - 2,
 		HeaderSize: 1024,
 	})
 	result, _ := list.Stat(func(hash string) bool {
 		// 随机测试
-		rand.Seed(time.Now().UnixNano())
 		return rand.Int()%2 == 0
 	})
 	t.Log(result)
 }
 
 func TestMemoryList_CleanPrefix(t *testing.T) {
-	list := NewMemoryList()
+	list := caches.NewMemoryList()
 	_ = list.Init()
 	before := time.Now()
-	for i := 0; i < 1_000_000; i++ {
+	var count = 100
+	if testutils.IsSingleTesting() {
+		count = 1_000_000
+	}
+	for i := 0; i < count; i++ {
 		key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
-		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
+		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
 			Key:        key,
 			ExpiredAt:  time.Now().Unix() + 3600,
 			BodySize:   0,
@@ -174,7 +184,12 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
 func TestMapRandomDelete(t *testing.T) {
 	var countMap = map[int]int{} // k => count
 
-	for j := 0; j < 1_000_000; j++ {
+	var count = 1000
+	if testutils.IsSingleTesting() {
+		count = 1_000_000
+	}
+
+	for j := 0; j < count; j++ {
 		var m = map[int]bool{}
 		for i := 0; i < 100; i++ {
 			m[i] = true
@@ -203,18 +218,18 @@ func TestMapRandomDelete(t *testing.T) {
 }
 
 func TestMemoryList_PurgeLFU(t *testing.T) {
-	var list = NewMemoryList().(*MemoryList)
+	var list = caches.NewMemoryList().(*caches.MemoryList)
 
 	var before = time.Now()
 	defer func() {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}()
 
-	_ = list.Add("1", &Item{})
-	_ = list.Add("2", &Item{})
-	_ = list.Add("3", &Item{})
-	_ = list.Add("4", &Item{})
-	_ = list.Add("5", &Item{})
+	_ = list.Add("1", &caches.Item{})
+	_ = list.Add("2", &caches.Item{})
+	_ = list.Add("3", &caches.Item{})
+	_ = list.Add("4", &caches.Item{})
+	_ = list.Add("5", &caches.Item{})
 
 	//_ = list.IncreaseHit("1")
 	//_ = list.IncreaseHit("2")
@@ -245,29 +260,32 @@ func TestMemoryList_PurgeLFU(t *testing.T) {
 }
 
 func TestMemoryList_CleanAll(t *testing.T) {
-	var list = NewMemoryList().(*MemoryList)
-	_ = list.Add("a", &Item{})
+	var list = caches.NewMemoryList().(*caches.MemoryList)
+	_ = list.Add("a", &caches.Item{})
 	_ = list.CleanAll()
-	logs.PrintAsJSON(list.itemMaps, t)
+	logs.PrintAsJSON(list.ItemMaps(), t)
 	t.Log(list.Count())
 }
 
 func TestMemoryList_GC(t *testing.T) {
-	list := NewMemoryList().(*MemoryList)
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	list := caches.NewMemoryList().(*caches.MemoryList)
 	_ = list.Init()
 	for i := 0; i < 1_000_000; i++ {
 		key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
-		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
+		_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &caches.Item{
 			Key:        key,
 			ExpiredAt:  0,
 			BodySize:   0,
 			HeaderSize: 0,
 		})
 	}
-	time.Sleep(10 * time.Second)
-	t.Log("clean...", len(list.itemMaps))
+	t.Log("clean...", len(list.ItemMaps()))
 	_ = list.CleanAll()
-	t.Log("cleanAll...", len(list.itemMaps))
+	t.Log("cleanAll...", len(list.ItemMaps()))
 	before := time.Now()
 	//runtime.GC()
 	t.Log("gc cost:", time.Since(before).Seconds()*1000, "ms")
@@ -280,3 +298,30 @@ func TestMemoryList_GC(t *testing.T) {
 		time.Sleep(30 * time.Minute)
 	}
 }
+
+func BenchmarkMemoryList(b *testing.B) {
+	var list = caches.NewMemoryList()
+	err := list.Init()
+	if err != nil {
+		b.Fatal(err)
+	}
+	for i := 0; i < 1_000_000; i++ {
+		_ = list.Add(stringutil.Md5(types.String(i)), &caches.Item{
+			Key:        "a1",
+			ExpiredAt:  time.Now().Unix() + 3600,
+			HeaderSize: 1024,
+		})
+	}
+
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, _ = list.Exist(types.String("a" + types.String(rands.Int(1, 10000))))
+			_ = list.Add("a"+types.String(rands.Int(1, 100000)), &caches.Item{})
+			_, _ = list.Purge(1000, func(hash string) error {
+				return nil
+			})
+		}
+	})
+}

internal/caches/manager.go

@@ -7,6 +7,7 @@ import (
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"golang.org/x/sys/unix"
@@ -35,6 +36,9 @@ type Manager struct {
 	SubDiskDirs       []*serverconfigs.CacheDir
 	MaxMemoryCapacity *shared.SizeCapacity
 
+	CountFileStorages   int
+	CountMemoryStorages int
+
 	policyMap  map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
 	storageMap map[int64]StorageInterface               // policyId => *Storage
 	locker     sync.RWMutex
@@ -143,6 +147,16 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
 			}
 		}
 	}
+
+	this.CountFileStorages = 0
+	this.CountFileStorages = 0
+	for _, storage := range this.storageMap {
+		_, isFileStorage := storage.(*FileStorage)
+		this.CountMemoryStorages++
+		if isFileStorage {
+			this.CountFileStorages++
+		}
+	}
 }
 
 // FindPolicy 获取Policy信息
@@ -172,6 +186,11 @@ func (this *Manager) NewStorageWithPolicy(policy *serverconfigs.HTTPCachePolicy)
 	return nil
 }
 
+// StorageMap 获取已有的存储对象
+func (this *Manager) StorageMap() map[int64]StorageInterface {
+	return this.storageMap
+}
+
 // TotalDiskSize 消耗的磁盘尺寸
 func (this *Manager) TotalDiskSize() int64 {
 	this.locker.RLock()
@@ -272,3 +291,17 @@ func (this *Manager) ScanGarbageCaches(callback func(path string) error) error {
 	}
 	return nil
 }
+
+// MaxSystemMemoryBytesPerStorage 计算单个策略能使用的系统最大内存
+func (this *Manager) MaxSystemMemoryBytesPerStorage() int64 {
+	var count = this.CountMemoryStorages
+	if count < 1 {
+		count = 1
+	}
+
+	var resultBytes = int64(utils.SystemMemoryBytes()) / 3 / int64(count) // 1/3 of the system memory
+	if resultBytes < 1<<30 {
+		resultBytes = 1 << 30
+	}
+	return resultBytes
+}

internal/caches/manager_test.go

@@ -1,8 +1,9 @@
-package caches
+package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/iwind/TeaGo/Tea"
 	"testing"
 )
@@ -10,7 +11,7 @@ import (
 func TestManager_UpdatePolicies(t *testing.T) {
 	{
 		var policies = []*serverconfigs.HTTPCachePolicy{}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 
@@ -38,7 +39,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
 				},
 			},
 		}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 
@@ -66,7 +67,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
 				},
 			},
 		}
-		SharedManager.UpdatePolicies(policies)
+		caches.SharedManager.UpdatePolicies(policies)
 		printManager(t)
 	}
 }
@@ -80,8 +81,8 @@ func TestManager_ChangePolicy_Memory(t *testing.T) {
 			Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
 		},
 	}
-	SharedManager.UpdatePolicies(policies)
-	SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
+	caches.SharedManager.UpdatePolicies(policies)
+	caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
 		{
 			Id:       1,
 			Type:     serverconfigs.CachePolicyStorageMemory,
@@ -102,8 +103,8 @@ func TestManager_ChangePolicy_File(t *testing.T) {
 			Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
 		},
 	}
-	SharedManager.UpdatePolicies(policies)
-	SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
+	caches.SharedManager.UpdatePolicies(policies)
+	caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
 		{
 			Id:   1,
 			Type: serverconfigs.CachePolicyStorageFile,
@@ -115,10 +116,17 @@ func TestManager_ChangePolicy_File(t *testing.T) {
 	})
 }
 
+func TestManager_MaxSystemMemoryBytesPerStorage(t *testing.T) {
+	for i := 0; i < 100; i++ {
+		caches.SharedManager.CountMemoryStorages = i
+		t.Log(i, caches.SharedManager.MaxSystemMemoryBytesPerStorage()>>30, "GB")
+	}
+}
+
 func printManager(t *testing.T) {
 	t.Log("===manager==")
 	t.Log("storage:")
-	for _, storage := range SharedManager.storageMap {
+	for _, storage := range caches.SharedManager.StorageMap() {
 		t.Log("  storage:", storage.Policy().Id)
 	}
 	t.Log("===============")

internal/caches/memory_fragment_pool.go

@@ -15,6 +15,7 @@ import (
 )
 
 const (
+	enableFragmentPool                  = false
 	minMemoryFragmentPoolItemSize       = 8 << 10
 	maxMemoryFragmentPoolItemSize       = 128 << 20
 	maxItemsInMemoryFragmentPoolBucket  = 1024

internal/caches/open_file_cache.go

@@ -93,7 +93,7 @@ func (this *OpenFileCache) Put(filename string, file *OpenFile) {
 	defer this.locker.Unlock()
 
 	// 如果超过当前容量，则关闭最早的
-	if this.count >= this.maxCount || this.usedSize >= this.capacitySize {
+	if this.count >= this.maxCount || this.usedSize+file.size >= this.capacitySize {
 		this.consumeHead()
 		return
 	}

internal/caches/storage_file.go

@@ -111,6 +111,15 @@ func (this *FileStorage) Policy() *serverconfigs.HTTPCachePolicy {
 
 // CanUpdatePolicy 检查策略是否可以更新
 func (this *FileStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool {
+	if newPolicy == nil {
+		return false
+	}
+
+	// 检查类型
+	if newPolicy.Type != serverconfigs.CachePolicyStorageFile {
+		return false
+	}
+
 	// 检查路径是否有变化
 	oldOptionsJSON, err := json.Marshal(this.policy.Options)
 	if err != nil {
@@ -455,7 +464,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	_, ok := sharedWritingFileKeyMap[key]
 	if ok {
 		sharedWritingFileKeyLocker.Unlock()
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w(001)", ErrFileIsWriting)
 	}
 
 	if !isFlushing && !fsutils.WriteReady() {
@@ -496,7 +505,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	// 检查两次写入缓存的时间是否过于相近，分片内容不受此限制
 	if err == nil && !isPartial && time.Since(stat.ModTime()) <= 1*time.Second {
 		// 防止并发连续写入
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w(002)", ErrFileIsWriting)
 	}
 
 	// 构造文件名
@@ -597,7 +606,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
 	err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
 	if err != nil {
 		removeOnFailure = false
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w (003)", ErrFileIsWriting)
 	}
 
 	var metaBodySize int64 = -1

internal/caches/storage_file_test.go

@@ -19,6 +19,10 @@ import (
 )
 
 func TestFileStorage_Init(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -49,6 +53,10 @@ func TestFileStorage_Init(t *testing.T) {
 }
 
 func TestFileStorage_OpenWriter(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -96,6 +104,10 @@ func TestFileStorage_OpenWriter(t *testing.T) {
 }
 
 func TestFileStorage_OpenWriter_Partial(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   2,
 		IsOn: true,
@@ -134,6 +146,10 @@ func TestFileStorage_OpenWriter_Partial(t *testing.T) {
 }
 
 func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -202,6 +218,10 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
 }
 
 func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -231,7 +251,7 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
 
 			writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, -1, false)
 			if err != nil {
-				if err != ErrFileIsWriting {
+				if errors.Is(err, ErrFileIsWriting) {
 					t.Error(err)
 					return
 				}
@@ -260,6 +280,10 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
 }
 
 func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -289,7 +313,7 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
 
 			writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, -1, false)
 			if err != nil {
-				if err != ErrFileIsWriting {
+				if errors.Is(err, ErrFileIsWriting) {
 					t.Error(err)
 					return
 				}
@@ -319,6 +343,10 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
 }
 
 func TestFileStorage_Read(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -358,6 +386,10 @@ func TestFileStorage_Read(t *testing.T) {
 }
 
 func TestFileStorage_Read_HTTP_Response(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -414,6 +446,10 @@ func TestFileStorage_Read_HTTP_Response(t *testing.T) {
 }
 
 func TestFileStorage_Read_NotFound(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -450,6 +486,10 @@ func TestFileStorage_Read_NotFound(t *testing.T) {
 }
 
 func TestFileStorage_Delete(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -472,6 +512,10 @@ func TestFileStorage_Delete(t *testing.T) {
 }
 
 func TestFileStorage_Stat(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -500,6 +544,10 @@ func TestFileStorage_Stat(t *testing.T) {
 }
 
 func TestFileStorage_CleanAll(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -534,6 +582,10 @@ func TestFileStorage_CleanAll(t *testing.T) {
 }
 
 func TestFileStorage_Stop(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -552,6 +604,10 @@ func TestFileStorage_Stop(t *testing.T) {
 }
 
 func TestFileStorage_DecodeFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
 		Id:   1,
 		IsOn: true,
@@ -571,6 +627,10 @@ func TestFileStorage_DecodeFile(t *testing.T) {
 }
 
 func TestFileStorage_RemoveCacheFile(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewFileStorage(nil)
 
 	defer storage.Stop()

internal/caches/storage_memory.go

@@ -1,6 +1,7 @@
 package caches
 
 import (
+	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -161,7 +162,7 @@ func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, h
 
 	// TODO 内存缓存暂时不支持分块内容存储
 	if isPartial {
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w (004)", ErrFileIsWriting)
 	}
 	return this.openWriter(key, expiredAt, status, headerSize, bodySize, maxSize, true)
 }
@@ -187,7 +188,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 	var isWriting = false
 	_, ok := this.writingKeyMap[key]
 	if ok {
-		return nil, ErrFileIsWriting
+		return nil, fmt.Errorf("%w (005)", ErrFileIsWriting)
 	}
 	this.writingKeyMap[key] = zero.New()
 	defer func() {
@@ -208,7 +209,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
 			_ = this.list.Remove(hashString)
 			item = nil
 		} else {
-			return nil, ErrFileIsWriting
+			return nil, fmt.Errorf("%w (006)", ErrFileIsWriting)
 		}
 	}
 
@@ -366,7 +367,7 @@ func (this *MemoryStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy
 
 // CanUpdatePolicy 检查策略是否可以更新
 func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool {
-	return true
+	return newPolicy != nil && newPolicy.Type == serverconfigs.CachePolicyStorageMemory
 }
 
 // AddToList 将缓存添加到列表
@@ -517,7 +518,7 @@ func (this *MemoryStorage) flushItem(key string) {
 		_ = this.Delete(key)
 
 		// 重用内存，前提是确保内存不再被引用
-		if ok && item.IsDone && !item.isReferring && len(item.BodyValue) > 0 {
+		if enableFragmentPool && ok && item.IsDone && !item.isReferring && len(item.BodyValue) > 0 {
 			SharedFragmentMemoryPool.Put(item.BodyValue)
 		}
 	}()
@@ -584,13 +585,18 @@ func (this *MemoryStorage) flushItem(key string) {
 }
 
 func (this *MemoryStorage) memoryCapacityBytes() int64 {
+	var maxSystemBytes = SharedManager.MaxSystemMemoryBytesPerStorage()
 	if this.policy == nil {
-		return 0
+		return maxSystemBytes
 	}
 
 	if SharedManager.MaxMemoryCapacity != nil {
 		var capacityBytes = SharedManager.MaxMemoryCapacity.Bytes()
 		if capacityBytes > 0 {
+			if capacityBytes > maxSystemBytes {
+				return maxSystemBytes
+			}
+
 			return capacityBytes
 		}
 	}
@@ -599,17 +605,14 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 {
 	if capacity != nil {
 		var capacityBytes = capacity.Bytes()
 		if capacityBytes > 0 {
+			if capacityBytes > maxSystemBytes {
+				return maxSystemBytes
+			}
 			return capacityBytes
 		}
 	}
 
-	// half of the system memory
-	var memoryGB = utils.SystemMemoryGB()
-	if memoryGB < 1 {
-		memoryGB = 1
-	}
-
-	return int64(memoryGB) << 30 / 2
+	return maxSystemBytes
 }
 
 func (this *MemoryStorage) deleteWithoutLocker(key string) error {

internal/caches/storage_memory_test.go

@@ -3,6 +3,7 @@ package caches
 import (
 	"bytes"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/rands"
 	"runtime"
@@ -271,6 +272,10 @@ func TestMemoryStorage_Purge(t *testing.T) {
 }
 
 func TestMemoryStorage_Expire(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
 		MemoryAutoPurgeInterval: 5,
 	}, nil)

internal/caches/writer_memory.go

@@ -32,7 +32,9 @@ func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64,
 		ModifiedAt: fasttime.Now().Unix(),
 		Status:     status,
 	}
-	if expectedBodySize > 0 && expectedBodySize <= maxMemoryFragmentPoolItemSize {
+	if enableFragmentPool &&
+		expectedBodySize > 0 &&
+		expectedBodySize <= maxMemoryFragmentPoolItemSize {
 		bodyBytes, ok := SharedFragmentMemoryPool.Get(expectedBodySize) // try to reuse memory
 		if ok {
 			valueItem.BodyValue = bodyBytes
@@ -123,7 +125,6 @@ func (this *MemoryWriter) Close() error {
 	// 需要在Locker之外
 	defer this.once.Do(func() {
 		this.endFunc(this.item)
-		this.item = nil // free memory
 	})
 
 	if this.item == nil {
@@ -162,13 +163,13 @@ func (this *MemoryWriter) Discard() error {
 	// 需要在Locker之外
 	defer this.once.Do(func() {
 		this.endFunc(this.item)
-		this.item = nil // free memory
 	})
 
 	this.storage.locker.Lock()
 	delete(this.storage.valuesMap, this.hash)
 
-	if this.item != nil &&
+	if enableFragmentPool &&
+		this.item != nil &&
 		!this.item.isReferring &&
 		cap(this.item.BodyValue) >= minMemoryFragmentPoolItemSize {
 		SharedFragmentMemoryPool.Put(this.item.BodyValue)

internal/compressions/reader_brotli.go

@@ -1,4 +1,5 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build !plus || !linux
 
 package compressions
 

internal/compressions/reader_brotli_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"github.com/TeaOSLab/EdgeNode/internal/compressions"
 	"io"
+	"os"
 	"testing"
 )
 
@@ -49,3 +50,45 @@ func TestBrotliReader(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkBrotliReader(b *testing.B) {
+	data, err := os.ReadFile("./reader_brotli.go")
+	if err != nil {
+		b.Fatal(err)
+	}
+	var buf = bytes.NewBuffer([]byte{})
+	writer, err := compressions.NewBrotliWriter(buf, 5)
+	if err != nil {
+		b.Fatal(err)
+	}
+	_, err = writer.Write(data)
+	err = writer.Close()
+	if err != nil {
+		b.Fatal(err)
+	}
+	var compressedData = buf.Bytes()
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			reader, readerErr := compressions.NewBrotliReader(bytes.NewBuffer(compressedData))
+			if readerErr != nil {
+				b.Fatal(readerErr)
+			}
+			var readBuf = make([]byte, 1024)
+			for {
+				_, readErr := reader.Read(readBuf)
+				if readErr != nil {
+					if readErr != io.EOF {
+						b.Fatal(readErr)
+					}
+					break
+				}
+			}
+			closeErr := reader.Close()
+			if closeErr != nil {
+				b.Fatal(closeErr)
+			}
+		}
+	})
+}

internal/compressions/utils.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"io"
+	"net/http"
 )
 
 type ContentEncoding = string
@@ -58,3 +59,32 @@ func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType,
 	}
 	return nil, errors.New("invalid compression type '" + compressType + "'")
 }
+
+// SupportEncoding 检查是否支持某个编码
+func SupportEncoding(encoding string) bool {
+	return encoding == ContentEncodingBr ||
+		encoding == ContentEncodingGzip ||
+		encoding == ContentEncodingDeflate ||
+		encoding == ContentEncodingZSTD
+}
+
+// WrapHTTPResponse 包装http.Response对象
+func WrapHTTPResponse(resp *http.Response) {
+	if resp == nil {
+		return
+	}
+
+	var contentEncoding = resp.Header.Get("Content-Encoding")
+	if len(contentEncoding) == 0 || !SupportEncoding(contentEncoding) {
+		return
+	}
+
+	reader, err := NewReader(resp.Body, contentEncoding)
+	if err != nil {
+		// unable to decode, we ignore the error
+		return
+	}
+	resp.Header.Del("Content-Encoding")
+	resp.Header.Del("Content-Length")
+	resp.Body = reader
+}

internal/compressions/writer_brotli.go

@@ -1,4 +1,5 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build !plus || !linux
 
 package compressions
 
@@ -27,7 +28,7 @@ func newBrotliWriter(writer io.Writer, level int) (*BrotliWriter, error) {
 	return &BrotliWriter{
 		writer: brotli.NewWriterOptions(writer, brotli.WriterOptions{
 			Quality: level,
-			LGWin:   13, // TODO 在全局设置里可以设置此值
+			LGWin:   14, // TODO 在全局设置里可以设置此值
 		}),
 		level: level,
 	}, nil

internal/compressions/writer_zstd.go

@@ -19,6 +19,10 @@ func NewZSTDWriter(writer io.Writer, level int) (Writer, error) {
 }
 
 func newZSTDWriter(writer io.Writer, level int) (Writer, error) {
+	if level < 0 {
+		level = 0
+	}
+
 	var zstdLevel = zstd.EncoderLevelFromZstd(level)
 
 	zstdWriter, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstdLevel))

internal/compressions/writer_zstd_test.go

@@ -9,6 +9,24 @@ import (
 	"testing"
 )
 
+func TestNewZSTDWriter_Level0(t *testing.T) {
+	var buf = &bytes.Buffer{}
+	writer, err := compressions.NewZSTDWriter(buf, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var originData = []byte(strings.Repeat("Hello", 1024))
+	_, err = writer.Write(originData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("origin data:", len(originData), "result:", buf.Len())
+}
+
 func TestNewZSTDWriter(t *testing.T) {
 	var buf = &bytes.Buffer{}
 	writer, err := compressions.NewZSTDWriter(buf, 10)

internal/configs/api_config.go

@@ -12,8 +12,8 @@ const oldConfigFileName = "api.yaml"
 
 type APIConfig struct {
 	OldRPC struct {
-		Endpoints     []string `yaml:"endpoints" json:"endpoints"`
-		DisableUpdate bool     `yaml:"disableUpdate" json:"disableUpdate"`
+		Endpoints     []string `yaml:"endpoints,omitempty" json:"endpoints"`
+		DisableUpdate bool     `yaml:"disableUpdate,omitempty" json:"disableUpdate"`
 	} `yaml:"rpc,omitempty" json:"rpc"`
 
 	RPCEndpoints     []string `yaml:"rpc.endpoints,flow" json:"rpc.endpoints"`

internal/configs/cluster_config_test.go

@@ -4,11 +4,16 @@ package configs_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/configs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"gopkg.in/yaml.v3"
 	"testing"
 )
 
 func TestLoadClusterConfig(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	config, err := configs.LoadClusterConfig()
 	if err != nil {
 		t.Fatal(err)

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "1.2.10"
+	Version = "1.3.3"
 
 	ProductName = "Edge Node"
 	ProcessName = "edge-node"

internal/iplibrary/action_http_api_test.go

@@ -3,11 +3,16 @@ package iplibrary
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 )
 
 func TestHTTPAPIAction_AddItem(t *testing.T) {
-	action := NewHTTPAPIAction()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var action = NewHTTPAPIAction()
 	action.config = &firewallconfigs.FirewallActionHTTPAPIConfig{
 		URL:            "http://127.0.0.1:2345/post",
 		TimeoutSeconds: 0,
@@ -24,7 +29,11 @@ func TestHTTPAPIAction_AddItem(t *testing.T) {
 }
 
 func TestHTTPAPIAction_DeleteItem(t *testing.T) {
-	action := NewHTTPAPIAction()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var action = NewHTTPAPIAction()
 	action.config = &firewallconfigs.FirewallActionHTTPAPIConfig{
 		URL:            "http://127.0.0.1:2345/post",
 		TimeoutSeconds: 0,

internal/iplibrary/action_ipset_test.go

@@ -4,13 +4,19 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/iwind/TeaGo/maps"
 	"testing"
 	"time"
 )
 
 func TestIPSetAction_Init(t *testing.T) {
-	action := iplibrary.NewIPSetAction()
+	_, lookupErr := executils.LookPath("iptables")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = iplibrary.NewIPSetAction()
 	err := action.Init(&firewallconfigs.FirewallActionConfig{
 		Params: maps.Map{
 			"path":      "/usr/bin/iptables",
@@ -25,6 +31,11 @@ func TestIPSetAction_Init(t *testing.T) {
 }
 
 func TestIPSetAction_AddItem(t *testing.T) {
+	_, lookupErr := executils.LookPath("iptables")
+	if lookupErr != nil {
+		return
+	}
+
 	var action = iplibrary.NewIPSetAction()
 	action.SetConfig(&firewallconfigs.FirewallActionIPSetConfig{
 		Path:          "/usr/bin/iptables",
@@ -84,7 +95,12 @@ func TestIPSetAction_AddItem(t *testing.T) {
 }
 
 func TestIPSetAction_DeleteItem(t *testing.T) {
-	action := iplibrary.NewIPSetAction()
+	_, lookupErr := executils.LookPath("firewalld")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = iplibrary.NewIPSetAction()
 	err := action.Init(&firewallconfigs.FirewallActionConfig{
 		Params: maps.Map{
 			"path":      "/usr/bin/firewalld",

internal/iplibrary/action_iptables_test.go

@@ -3,12 +3,18 @@ package iplibrary
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"testing"
 	"time"
 )
 
 func TestIPTablesAction_AddItem(t *testing.T) {
-	action := NewIPTablesAction()
+	_, lookupErr := executils.LookPath("iptables")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = NewIPTablesAction()
 	action.config = &firewallconfigs.FirewallActionIPTablesConfig{
 		Path: "/usr/bin/iptables",
 	}
@@ -40,7 +46,12 @@ func TestIPTablesAction_AddItem(t *testing.T) {
 }
 
 func TestIPTablesAction_DeleteItem(t *testing.T) {
-	action := NewIPTablesAction()
+	_, lookupErr := executils.LookPath("firewalld")
+	if lookupErr != nil {
+		return
+	}
+
+	var action = NewIPTablesAction()
 	action.config = &firewallconfigs.FirewallActionIPTablesConfig{
 		Path: "/usr/bin/firewalld",
 	}

internal/iplibrary/action_manager_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestActionManager_UpdateActions(t *testing.T) {
-	manager := NewActionManager()
+	var manager = NewActionManager()
 	manager.UpdateActions([]*firewallconfigs.FirewallActionConfig{
 		{
 			Id:   1,

internal/iplibrary/action_script_test.go

@@ -3,11 +3,16 @@ package iplibrary
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 	"time"
 )
 
 func TestScriptAction_AddItem(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	action := NewScriptAction()
 	action.config = &firewallconfigs.FirewallActionScriptConfig{
 		Path: "/tmp/ip-item.sh",
@@ -27,6 +32,10 @@ func TestScriptAction_AddItem(t *testing.T) {
 }
 
 func TestScriptAction_DeleteItem(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	action := NewScriptAction()
 	action.config = &firewallconfigs.FirewallActionScriptConfig{
 		Path: "/tmp/ip-item.sh",

internal/iplibrary/ip_item_test.go

@@ -2,6 +2,7 @@ package iplibrary
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
 	"runtime"
 	"testing"
@@ -75,8 +76,14 @@ func TestIPItem_Contains(t *testing.T) {
 }
 
 func TestIPItem_Memory(t *testing.T) {
+	var isSingleTest = testutils.IsSingleTesting()
+
 	var list = NewIPList()
-	for i := 0; i < 2_000_000; i ++ {
+	var count = 100
+	if isSingleTest {
+		count = 2_000_000
+	}
+	for i := 0; i < count; i++ {
 		list.Add(&IPItem{
 			Type:       "ip",
 			Id:         uint64(i),
@@ -87,7 +94,9 @@ func TestIPItem_Memory(t *testing.T) {
 		})
 	}
 	t.Log("waiting")
-	time.Sleep(10 * time.Second)
+	if isSingleTest {
+		time.Sleep(10 * time.Second)
+	}
 }
 
 func BenchmarkIPItem_Contains(b *testing.B) {
@@ -105,4 +114,3 @@ func BenchmarkIPItem_Contains(b *testing.B) {
 		}
 	}
 }
-

internal/iplibrary/ip_list_db_test.go

@@ -16,6 +16,9 @@ func TestIPListDB_AddItem(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	err = db.AddItem(&pb.IPItem{
 		Id:                            1,
@@ -60,6 +63,9 @@ func TestIPListDB_ReadItems(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	defer func() {
 		_ = db.Close()
@@ -77,6 +83,9 @@ func TestIPListDB_ReadMaxVersion(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 	t.Log(db.ReadMaxVersion())
 }
 
@@ -85,6 +94,10 @@ func TestIPListDB_UpdateMaxVersion(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
+
 	err = db.UpdateMaxVersion(1027)
 	if err != nil {
 		t.Fatal(err)

internal/iplibrary/list_utils_test.go

@@ -3,12 +3,17 @@
 package iplibrary
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 	"time"
 )
 
 func TestIPIsAllowed(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var manager = NewIPListManager()
 	manager.init()
 
 	var before = time.Now()

internal/iplibrary/manager_ip_list_test.go

@@ -2,13 +2,18 @@ package iplibrary
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/logs"
 	"testing"
 	"time"
 )
 
 func TestIPListManager_init(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var manager = NewIPListManager()
 	manager.init()
 	t.Log(manager.listMap)
 	t.Log(SharedServerListManager.blackMap)
@@ -16,7 +21,11 @@ func TestIPListManager_init(t *testing.T) {
 }
 
 func TestIPListManager_check(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var manager = NewIPListManager()
 	manager.init()
 
 	var before = time.Now()
@@ -28,7 +37,11 @@ func TestIPListManager_check(t *testing.T) {
 }
 
 func TestIPListManager_loop(t *testing.T) {
-	manager := NewIPListManager()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var  manager = NewIPListManager()
 	manager.Start()
 	err := manager.loop()
 	if err != nil {

internal/monitor/value_queue_test.go

@@ -5,6 +5,7 @@ package monitor
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/logs"
 	"google.golang.org/grpc/status"
@@ -12,6 +13,10 @@ import (
 )
 
 func TestValueQueue_RPC(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	rpcClient, err := rpc.SharedRPC()
 	if err != nil {
 		t.Fatal(err)

internal/nodes/api_stream.go

@@ -90,13 +90,13 @@ func (this *APIStream) loop() error {
 			break
 		}
 
-		message, err := nodeStream.Recv()
-		if err != nil {
+		message, streamErr := nodeStream.Recv()
+		if streamErr != nil {
 			if this.isQuiting {
 				remotelogs.Println("API_STREAM", "quit")
 				return nil
 			}
-			return err
+			return streamErr
 		}
 
 		// 处理消息

internal/nodes/api_stream_test.go

@@ -1,8 +1,15 @@
 package nodes
 
-import "testing"
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"testing"
+)
 
 func TestAPIStream_Start(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	apiStream := NewAPIStream()
 	apiStream.Start()
 }

internal/nodes/client_conn.go

@@ -305,7 +305,7 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
 			// 非TLS，设置为两倍，防止误封
 			minAttempts = 2 * minAttempts
 		}
-		if result >= types.Uint64(minAttempts) {
+		if result >= types.Uint32(minAttempts) {
 			var timeout = synFloodConfig.TimeoutSeconds
 			if timeout <= 0 {
 				timeout = 600

internal/nodes/http_access_log_queue_test.go

@@ -110,6 +110,10 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
 }
 
 func TestHTTPAccessLogQueue_Memory(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	testutils.StartMemoryStats(t)
 
 	debug.SetGCPercent(10)

internal/nodes/http_client_pool_test.go

@@ -3,13 +3,14 @@ package nodes
 import (
 	"context"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"runtime"
 	"testing"
 	"time"
 )
 
 func TestHTTPClientPool_Client(t *testing.T) {
-	pool := NewHTTPClientPool()
+	var pool = NewHTTPClientPool()
 
 	{
 		var origin = &serverconfigs.OriginConfig{
@@ -54,7 +55,10 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		t.Log("get", i)
 		_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil, false)
-		time.Sleep(1 * time.Second)
+
+		if testutils.IsSingleTesting() {
+			time.Sleep(1 * time.Second)
+		}
 	}
 }
 

internal/nodes/http_request.go

@@ -85,6 +85,8 @@ type HTTPRequest struct {
 	isAttack        bool   // 是否是攻击请求
 	requestBodyData []byte // 读取的Body内容
 
+	isWebsocketResponse bool // 是否为Websocket响应（非请求）
+
 	// WAF相关
 	firewallPolicyId    int64
 	firewallRuleGroupId int64
@@ -100,6 +102,8 @@ type HTTPRequest struct {
 	disableLog bool // 是否在当前请求中关闭Log
 	forceLog   bool // 是否强制记录日志
 
+	isHijacked bool
+
 	// script相关操作
 	isDone bool
 }
@@ -191,17 +195,35 @@ func (this *HTTPRequest) Do() {
 		}
 
 		// 套餐
-		if this.ReqServer.UserPlan != nil && !this.ReqServer.UserPlan.IsAvailable() {
-			this.doPlanExpires()
-			this.doEnd()
-			return
+		if this.ReqServer.UserPlan != nil {
+			if this.doPlanBefore() {
+				this.doEnd()
+				return
+			}
 		}
 
 		// 流量限制
 		if this.ReqServer.TrafficLimitStatus != nil && this.ReqServer.TrafficLimitStatus.IsValid() {
-			this.doTrafficLimit()
-			this.doEnd()
-			return
+			if this.doTrafficLimit(this.ReqServer.TrafficLimitStatus) {
+				this.doEnd()
+				return
+			}
+		}
+
+		// UAM
+		var uamIsCalled = false
+		if !this.isHealthCheck {
+			if this.web.UAM == nil && this.ReqServer.UAM != nil && this.ReqServer.UAM.IsOn {
+				this.web.UAM = this.ReqServer.UAM
+			}
+
+			if this.web.UAM != nil && this.web.UAM.IsOn && this.isUAMRequest() {
+				uamIsCalled = true
+				if this.doUAM() {
+					this.doEnd()
+					return
+				}
+			}
 		}
 
 		// WAF
@@ -213,16 +235,8 @@ func (this *HTTPRequest) Do() {
 		}
 
 		// UAM
-		if !this.isHealthCheck {
-			if this.web.UAM != nil {
-				if this.web.UAM.IsOn {
-					if this.doUAM() {
-						this.doEnd()
-						return
-					}
-				}
-			} else if this.ReqServer.UAM != nil && this.ReqServer.UAM.IsOn {
-				this.web.UAM = this.ReqServer.UAM
+		if !this.isHealthCheck && !uamIsCalled {
+			if this.web.UAM != nil && this.web.UAM.IsOn {
 				if this.doUAM() {
 					this.doEnd()
 					return
@@ -278,6 +292,16 @@ func (this *HTTPRequest) Do() {
 		if this.web.Compression != nil && this.web.Compression.IsOn && this.web.Compression.Level > 0 {
 			this.writer.SetCompression(this.web.Compression)
 		}
+
+		// HLS
+		if this.web.HLS != nil &&
+			this.web.HLS.Encrypting != nil &&
+			this.web.HLS.Encrypting.IsOn {
+			if this.processHLSBefore() {
+				this.doEnd()
+				return
+			}
+		}
 	}
 
 	// 开始调用
@@ -410,6 +434,8 @@ func (this *HTTPRequest) doEnd() {
 		var countAttacks int64 = 0
 		var attackBytes int64 = 0
 
+		var countWebsocketConnections int64 = 0
+
 		if this.isCached {
 			countCached = 1
 			cachedBytes = totalBytes
@@ -421,8 +447,11 @@ func (this *HTTPRequest) doEnd() {
 				attackBytes = totalBytes
 			}
 		}
+		if this.isWebsocketResponse {
+			countWebsocketConnections = 1
+		}
 
-		stats.SharedTrafficStatManager.Add(this.ReqServer.UserId, this.ReqServer.Id, this.ReqHost, totalBytes, cachedBytes, 1, countCached, countAttacks, attackBytes, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
+		stats.SharedTrafficStatManager.Add(this.ReqServer.UserId, this.ReqServer.Id, this.ReqHost, totalBytes, cachedBytes, 1, countCached, countAttacks, attackBytes, countWebsocketConnections, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
 
 		// 指标
 		if metrics.SharedManager.HasHTTPMetrics() {
@@ -624,6 +653,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
 		this.web.CC = web.CC
 	}
 
+	// HLS
+	if web.HLS != nil && (web.HLS.IsPrior || isTop) {
+		this.web.HLS = web.HLS
+	}
+
 	// 重写规则
 	if len(web.RewriteRefs) > 0 {
 		for index, ref := range web.RewriteRefs {
@@ -1423,11 +1457,25 @@ func (this *HTTPRequest) requestScheme() string {
 
 // 请求的服务器地址中的端口
 func (this *HTTPRequest) requestServerPort() int {
-	_, port, err := net.SplitHostPort(this.ServerAddr)
-	if err == nil {
-		return types.Int(port)
+	if len(this.ServerAddr) > 0 {
+		_, port, err := net.SplitHostPort(this.ServerAddr)
+		if err == nil && len(port) > 0 {
+			return types.Int(port)
+		}
 	}
-	return 0
+
+	var host = this.RawReq.Host
+	if len(host) > 0 {
+		_, port, err := net.SplitHostPort(host)
+		if err == nil && len(port) > 0 {
+			return types.Int(port)
+		}
+	}
+
+	if this.IsHTTP {
+		return 80
+	}
+	return 443
 }
 
 func (this *HTTPRequest) Id() string {

internal/nodes/http_request_auth.go

@@ -43,7 +43,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 			if uriChanged {
 				this.uri = newURI
 			}
-			this.tags = append(this.tags, ref.AuthPolicy.Type)
+			this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
 			return
 		} else {
 			// Basic Auth比较特殊
@@ -64,7 +64,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 				}
 			}
 			this.writer.WriteHeader(http.StatusUnauthorized)
-			this.tags = append(this.tags, ref.AuthPolicy.Type)
+			this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
 			return true
 		}
 	}

internal/nodes/http_request_cache.go

@@ -3,6 +3,7 @@ package nodes
 import (
 	"bytes"
 	"errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/compressions"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -130,7 +131,22 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	var tags = []string{}
 
 	// 检查是否有缓存
-	var key = this.Format(this.cacheRef.Key)
+	var key string
+	if this.web.Cache.Key != nil && this.web.Cache.Key.IsOn && len(this.web.Cache.Key.Host) > 0 {
+		key = configutils.ParseVariables(this.cacheRef.Key, func(varName string) (value string) {
+			switch varName {
+			case "scheme":
+				return this.web.Cache.Key.Scheme
+			case "host":
+				return this.web.Cache.Key.Host
+			default:
+				return this.Format("${" + varName + "}")
+			}
+		})
+	} else {
+		key = this.Format(this.cacheRef.Key)
+	}
+
 	if len(key) == 0 {
 		this.cacheRef = nil
 		cacheBypassDescription = "BYPASS, empty key"
@@ -274,7 +290,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		}
 
 		if err != nil {
-			if err == caches.ErrNotFound {
+			if errors.Is(err, caches.ErrNotFound) {
+				// 移除请求中的 If-None-Match 和 If-Modified-Since，防止源站返回304而无法缓存
+				if this.reverseProxy != nil {
+					this.RawReq.Header.Del("If-None-Match")
+					this.RawReq.Header.Del("If-Modified-Since")
+				}
+
 				// cache相关变量
 				this.varMapping["cache.status"] = "MISS"
 
@@ -365,24 +387,24 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	}
 
 	// ETag
-	// 这里强制设置ETag，如果先前源站设置了ETag，将会被覆盖，避免因为源站的ETag导致源站返回304 Not Modified
 	var respHeader = this.writer.Header()
-	var eTag = ""
+	var eTag = respHeader.Get("ETag")
 	var lastModifiedAt = reader.LastModified()
-	if lastModifiedAt > 0 {
-		if len(tags) > 0 {
-			eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
-		} else {
-			eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
-		}
-		respHeader.Del("Etag")
-		if !isPartialCache {
-			respHeader["ETag"] = []string{eTag}
+	if len(eTag) == 0 {
+		if lastModifiedAt > 0 {
+			if len(tags) > 0 {
+				eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
+			} else {
+				eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
+			}
+			respHeader.Del("Etag")
+			if !isPartialCache {
+				respHeader["ETag"] = []string{eTag}
+			}
 		}
 	}
 
 	// 支持 Last-Modified
-	// 这里强制设置Last-Modified，如果先前源站设置了Last-Modified，将会被覆盖，避免因为源站的Last-Modified导致源站返回304 Not Modified
 	var modifiedTime = ""
 	if lastModifiedAt > 0 {
 		modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
@@ -490,7 +512,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 			if err != nil {
 				this.varMapping["cache.status"] = "MISS"
 
-				if err == caches.ErrInvalidRange {
+				if errors.Is(err, caches.ErrInvalidRange) {
 					this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
 					this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 					return true

internal/nodes/http_request_hls.go

@@ -0,0 +1,16 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+//go:build !plus
+
+package nodes
+
+import "net/http"
+
+func (this *HTTPRequest) processHLSBefore() (blocked bool) {
+	//  stub
+	return false
+}
+
+func (this *HTTPRequest) processM3u8Response(resp *http.Response) error {
+	// stub
+	return nil
+}

internal/nodes/http_request_host_redirect.go

@@ -25,6 +25,13 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 			continue
 		}
 
+		if len(u.ExceptDomains) > 0 && configutils.MatchDomains(u.ExceptDomains, this.ReqHost) {
+			continue
+		}
+		if len(u.OnlyDomains) > 0 && !configutils.MatchDomains(u.OnlyDomains, this.ReqHost) {
+			continue
+		}
+
 		var status = u.Status
 		if status <= 0 {
 			if searchEngineRegex.MatchString(this.RawReq.UserAgent()) {

internal/nodes/http_request_mismatch.go

@@ -34,60 +34,63 @@ func (this *HTTPRequest) doMismatch() {
 	}
 
 	// 根据配置进行相应的处理
-	var globalServerConfig = sharedNodeConfig.GlobalServerConfig
-	if globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
-		var statusCode = 404
-		var httpAllConfig = globalServerConfig.HTTPAll
-		var mismatchAction = httpAllConfig.DomainMismatchAction
+	var nodeConfig = sharedNodeConfig // copy
+	if nodeConfig != nil {
+		var globalServerConfig = nodeConfig.GlobalServerConfig
+		if globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
+			var statusCode = 404
+			var httpAllConfig = globalServerConfig.HTTPAll
+			var mismatchAction = httpAllConfig.DomainMismatchAction
 
-		if mismatchAction != nil && mismatchAction.Options != nil {
-			var mismatchStatusCode = mismatchAction.Options.GetInt("statusCode")
-			if mismatchStatusCode > 0 && mismatchStatusCode >= 100 && mismatchStatusCode < 1000 {
-				statusCode = mismatchStatusCode
-			}
-		}
-
-		// 是否正在访问IP
-		if globalServerConfig.HTTPAll.NodeIPShowPage && utils.IsWildIP(this.ReqHost) {
-			this.writer.statusCode = statusCode
-			var contentHTML = this.Format(globalServerConfig.HTTPAll.NodeIPPageHTML)
-			this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
-			this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
-			this.writer.WriteHeader(statusCode)
-			_, _ = this.writer.WriteString(contentHTML)
-			return
-		}
-
-		// 检查cc
-		// TODO 可以在管理端配置是否开启以及最多尝试次数
-		// 要考虑到服务在切换集群时，域名未生效状态时，用户访问的仍然是老集群中的节点，就会产生找不到域名的情况
-		if len(remoteIP) > 0 {
-			const maxAttempts = 100
-			if ttlcache.SharedInt64Cache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
-				// 在加入之前再次检查黑名单
-				if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
-					waf.SharedIPBlackList.Add(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+3600)
+			if mismatchAction != nil && mismatchAction.Options != nil {
+				var mismatchStatusCode = mismatchAction.Options.GetInt("statusCode")
+				if mismatchStatusCode > 0 && mismatchStatusCode >= 100 && mismatchStatusCode < 1000 {
+					statusCode = mismatchStatusCode
 				}
 			}
-		}
 
-		// 处理当前连接
-		if mismatchAction != nil && mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
-			if mismatchAction.Options != nil {
+			// 是否正在访问IP
+			if globalServerConfig.HTTPAll.NodeIPShowPage && utils.IsWildIP(this.ReqHost) {
 				this.writer.statusCode = statusCode
-				var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
+				var contentHTML = this.Format(globalServerConfig.HTTPAll.NodeIPPageHTML)
 				this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 				this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
 				this.writer.WriteHeader(statusCode)
-				_, _ = this.writer.Write([]byte(contentHTML))
+				_, _ = this.writer.WriteString(contentHTML)
+				return
+			}
+
+			// 检查cc
+			// TODO 可以在管理端配置是否开启以及最多尝试次数
+			// 要考虑到服务在切换集群时，域名未生效状态时，用户访问的仍然是老集群中的节点，就会产生找不到域名的情况
+			if len(remoteIP) > 0 {
+				const maxAttempts = 100
+				if ttlcache.SharedInt64Cache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
+					// 在加入之前再次检查黑名单
+					if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
+						waf.SharedIPBlackList.Add(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+3600)
+					}
+				}
+			}
+
+			// 处理当前连接
+			if mismatchAction != nil && mismatchAction.Code == serverconfigs.DomainMismatchActionPage {
+				if mismatchAction.Options != nil {
+					this.writer.statusCode = statusCode
+					var contentHTML = this.Format(mismatchAction.Options.GetString("contentHTML"))
+					this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+					this.writer.Header().Set("Content-Length", types.String(len(contentHTML)))
+					this.writer.WriteHeader(statusCode)
+					_, _ = this.writer.Write([]byte(contentHTML))
+				} else {
+					http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
+				}
+				return
 			} else {
 				http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
+				this.Close()
+				return
 			}
-			return
-		} else {
-			http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
-			this.Close()
-			return
 		}
 	}
 

internal/nodes/http_request_page.go

@@ -2,7 +2,6 @@ package nodes
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/Tea"
@@ -46,9 +45,13 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
 }
 
 func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, status int) (shouldStop bool) {
+	var url = this.URL()
 	for _, page := range pages {
+		if !page.MatchURL(url) {
+			continue
+		}
 		if page.Match(status) {
-			if len(page.BodyType) == 0 || page.BodyType == shared.BodyTypeURL {
+			if len(page.BodyType) == 0 || page.BodyType == serverconfigs.HTTPPageBodyTypeURL {
 				if urlSchemeRegexp.MatchString(page.URL) {
 					var newStatus = page.NewStatus
 					if newStatus <= 0 {
@@ -115,7 +118,7 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
 				}
 
 				return true
-			} else if page.BodyType == shared.BodyTypeHTML {
+			} else if page.BodyType == serverconfigs.HTTPPageBodyTypeHTML {
 				// 这里需要实现设置Status，因为在Format()中可以获取${status}等变量
 				if page.NewStatus > 0 {
 					this.writer.statusCode = page.NewStatus
@@ -147,6 +150,18 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
 					this.writer.SetOk()
 				}
 				return true
+			} else if page.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
+				var newURL = this.Format(page.URL)
+				if len(newURL) == 0 {
+					newURL = "/"
+				}
+				if page.NewStatus > 0 && httpStatusIsRedirect(page.NewStatus) {
+					httpRedirect(this.writer, this.RawReq, newURL, page.NewStatus)
+				} else {
+					httpRedirect(this.writer, this.RawReq, newURL, http.StatusTemporaryRedirect)
+				}
+				this.writer.SetOk()
+				return true
 			}
 		}
 	}

internal/nodes/http_request_plan_before.go

@@ -0,0 +1,10 @@
+// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build !plus
+
+package nodes
+
+// 检查套餐
+func (this *HTTPRequest) doPlanBefore() (blocked bool) {
+	// stub
+	return false
+}

internal/nodes/http_request_plan_expires.go

@@ -1,19 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package nodes
-
-import (
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"net/http"
-)
-
-// 套餐过期
-func (this *HTTPRequest) doPlanExpires() {
-	this.tags = append(this.tags, "plan")
-
-	var statusCode = http.StatusNotFound
-	this.ProcessResponseHeaders(this.writer.Header(), statusCode)
-
-	this.writer.WriteHeader(statusCode)
-	_, _ = this.writer.WriteString(this.Format(serverconfigs.DefaultPlanExpireNoticePageBody))
-}

internal/nodes/http_request_reverse_proxy.go

@@ -27,9 +27,10 @@ func (this *HTTPRequest) doReverseProxy() {
 
 	var failedOriginIds []int64
 	var failedLnNodeIds []int64
+	var failStatusCode int
 
 	for i := 0; i < retries; i++ {
-		originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1)
+		originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1, &failStatusCode)
 		if !shouldRetry {
 			break
 		}
@@ -43,7 +44,7 @@ func (this *HTTPRequest) doReverseProxy() {
 }
 
 // 请求源站
-func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool) (originId int64, lnNodeId int64, shouldRetry bool) {
+func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool, failStatusCode *int) (originId int64, lnNodeId int64, shouldRetry bool) {
 	// 对URL的处理
 	var stripPrefix = this.reverseProxy.StripPrefix
 	var requestURI = this.reverseProxy.RequestURI
@@ -91,6 +92,10 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 		}
 		if origin == nil {
 			origin = this.reverseProxy.NextOrigin(requestCall)
+			if origin != nil && origin.Id > 0 && (*failStatusCode >= 403 && *failStatusCode <= 404) && lists.ContainsInt64(failedOriginIds, origin.Id) {
+				this.writeCode(*failStatusCode, "", "")
+				return
+			}
 		}
 		requestCall.CallResponseCallbacks(this.writer)
 		if origin == nil {
@@ -376,11 +381,20 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 		return
 	}
 
-	// 50x
-	if resp != nil &&
-		resp.StatusCode >= 500 &&
-		resp.StatusCode < 510 &&
-		this.reverseProxy.Retry50X &&
+	if resp == nil {
+		this.write50x(requestErr, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
+		return
+	}
+
+	// fix Content-Type
+	if resp.Header["Content-Type"] == nil {
+		resp.Header["Content-Type"] = []string{}
+	}
+
+	// 40x && 50x
+	*failStatusCode = resp.StatusCode
+	if ((resp.StatusCode >= 500 && resp.StatusCode < 510 && this.reverseProxy.Retry50X) ||
+		(resp.StatusCode >= 403 && resp.StatusCode <= 404 && this.reverseProxy.Retry40X)) &&
 		(originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) &&
 		!isLastRetry {
 		if resp.Body != nil {
@@ -392,8 +406,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 	}
 
 	// 尝试从缓存中恢复
-	if resp != nil &&
-		resp.StatusCode >= 500 && // support 50X only
+	if resp.StatusCode >= 500 && // support 50X only
 		resp.StatusCode < 510 &&
 		this.cacheCanTryStale &&
 		this.web.Cache.Stale != nil &&
@@ -429,21 +442,43 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
 
 	// Page optimization
 	if this.web.Optimization != nil && resp.Body != nil && this.cacheRef != nil /** must under cache **/ {
-		err := this.web.Optimization.FilterResponse(resp)
+		err := this.web.Optimization.FilterResponse(this.URL(), resp)
 		if err != nil {
-			this.write50x(err, http.StatusBadGateway, "Page Optimization: Fail to read content from origin", "内容优化：从源站读取内容失败", false)
+			this.write50x(err, http.StatusBadGateway, "Page Optimization: fail to read content from origin", "内容优化：从源站读取内容失败", false)
+			return
+		}
+	}
+
+	// HLS
+	if this.web.HLS != nil &&
+		this.web.HLS.Encrypting != nil &&
+		this.web.HLS.Encrypting.IsOn &&
+		resp.StatusCode == http.StatusOK {
+		m3u8Err := this.processM3u8Response(resp)
+		if m3u8Err != nil {
+			this.write50x(m3u8Err, http.StatusBadGateway, "m3u8 encrypt: fail to read content from origin", "m3u8加密：从源站读取内容失败", false)
 			return
 		}
 	}
 
 	// 设置Charset
-	// TODO 这里应该可以设置文本类型的列表，以及是否强制覆盖所有文本类型的字符集
+	// TODO 这里应该可以设置文本类型的列表
 	if this.web.Charset != nil && this.web.Charset.IsOn && len(this.web.Charset.Charset) > 0 {
 		contentTypes, ok := resp.Header["Content-Type"]
 		if ok && len(contentTypes) > 0 {
 			var contentType = contentTypes[0]
+			if this.web.Charset.Force {
+				var semiIndex = strings.Index(contentType, ";")
+				if semiIndex > 0 {
+					contentType = contentType[:semiIndex]
+				}
+			}
 			if _, found := textMimeMap[contentType]; found {
-				resp.Header["Content-Type"][0] = contentType + "; charset=" + this.web.Charset.Charset
+				var newCharset = this.web.Charset.Charset
+				if this.web.Charset.IsUpper {
+					newCharset = strings.ToUpper(newCharset)
+				}
+				resp.Header["Content-Type"][0] = contentType + "; charset=" + newCharset
 			}
 		}
 	}

internal/nodes/http_request_shutdown.go

@@ -1,7 +1,7 @@
 package nodes
 
 import (
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/Tea"
@@ -18,7 +18,7 @@ func (this *HTTPRequest) doShutdown() {
 		return
 	}
 
-	if len(shutdown.BodyType) == 0 || shutdown.BodyType == shared.BodyTypeURL {
+	if len(shutdown.BodyType) == 0 || shutdown.BodyType == serverconfigs.HTTPPageBodyTypeURL {
 		// URL
 		if urlSchemeRegexp.MatchString(shutdown.URL) {
 			this.doURL(http.MethodGet, shutdown.URL, "", shutdown.Status, true)
@@ -80,7 +80,7 @@ func (this *HTTPRequest) doShutdown() {
 		} else {
 			this.writer.SetOk()
 		}
-	} else if shutdown.BodyType == shared.BodyTypeHTML {
+	} else if shutdown.BodyType == serverconfigs.HTTPPageBodyTypeHTML {
 		// 自定义响应Headers
 		if shutdown.Status > 0 {
 			this.ProcessResponseHeaders(this.writer.Header(), shutdown.Status)
@@ -98,5 +98,17 @@ func (this *HTTPRequest) doShutdown() {
 		} else {
 			this.writer.SetOk()
 		}
+	} else if shutdown.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
+		var newURL = shutdown.URL
+		if len(newURL) == 0 {
+			newURL = "/"
+		}
+
+		if shutdown.Status > 0 && httpStatusIsRedirect(shutdown.Status) {
+			httpRedirect(this.writer, this.RawReq, newURL, shutdown.Status)
+		} else {
+			httpRedirect(this.writer, this.RawReq, newURL, http.StatusTemporaryRedirect)
+		}
+		this.writer.SetOk()
 	}
 }

internal/nodes/http_request_test.go

@@ -3,6 +3,7 @@ package nodes
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/iwind/TeaGo/assert"
+	"net/http"
 	"runtime"
 	"testing"
 )
@@ -10,26 +11,45 @@ import (
 func TestHTTPRequest_RedirectToHTTPS(t *testing.T) {
 	var a = assert.NewAssertion(t)
 	{
-		req := &HTTPRequest{
+		rawReq, err := http.NewRequest(http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var req = &HTTPRequest{
+			RawReq:    rawReq,
+			RawWriter: NewEmptyResponseWriter(nil),
 			ReqServer: &serverconfigs.ServerConfig{
+				IsOn: true,
 				Web: &serverconfigs.HTTPWebConfig{
+					IsOn:            true,
 					RedirectToHttps: &serverconfigs.HTTPRedirectToHTTPSConfig{},
 				},
 			},
 		}
+		req.init()
 		req.Do()
+
 		a.IsBool(req.web.RedirectToHttps.IsOn == false)
 	}
 	{
-		req := &HTTPRequest{
+		rawReq, err := http.NewRequest(http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var req = &HTTPRequest{
+			RawReq:    rawReq,
+			RawWriter: NewEmptyResponseWriter(nil),
 			ReqServer: &serverconfigs.ServerConfig{
+				IsOn: true,
 				Web: &serverconfigs.HTTPWebConfig{
+					IsOn: true,
 					RedirectToHttps: &serverconfigs.HTTPRedirectToHTTPSConfig{
 						IsOn: true,
 					},
 				},
 			},
 		}
+		req.init()
 		req.Do()
 		a.IsBool(req.web.RedirectToHttps.IsOn == true)
 	}

internal/nodes/http_request_traffic_limit.go

@@ -7,7 +7,19 @@ import (
 )
 
 // 流量限制
-func (this *HTTPRequest) doTrafficLimit() {
+func (this *HTTPRequest) doTrafficLimit(status *serverconfigs.TrafficLimitStatus) (blocked bool) {
+	if status == nil {
+		return false
+	}
+
+	// 如果是网站单独设置的流量限制，则检查是否已关闭
+	var config = this.ReqServer.TrafficLimit
+	if (config == nil || !config.IsOn) && status.PlanId == 0 {
+		return false
+	}
+
+	// 如果是套餐设置的流量限制，即使套餐变更了（变更套餐或者变更套餐的限制），仍然会提示流量超限
+
 	this.tags = append(this.tags, "trafficLimit")
 
 	var statusCode = 509
@@ -17,10 +29,19 @@ func (this *HTTPRequest) doTrafficLimit() {
 	this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 	this.writer.WriteHeader(statusCode)
 
-	var config = this.ReqServer.TrafficLimit
+	// check plan traffic limit
+	if (config == nil || !config.IsOn) && this.ReqServer.PlanId() > 0 && this.nodeConfig != nil {
+		var planConfig = this.nodeConfig.FindPlan(this.ReqServer.PlanId())
+		if planConfig != nil && planConfig.TrafficLimit != nil && planConfig.TrafficLimit.IsOn {
+			config = planConfig.TrafficLimit
+		}
+	}
+
 	if config != nil && len(config.NoticePageBody) != 0 {
 		_, _ = this.writer.WriteString(this.Format(config.NoticePageBody))
 	} else {
 		_, _ = this.writer.WriteString(this.Format(serverconfigs.DefaultTrafficLimitNoticePageBody))
 	}
+
+	return true
 }

internal/nodes/http_request_uam.go

@@ -4,7 +4,13 @@
 
 package nodes
 
-// UAM
-func (this *HTTPRequest) doUAM() (block bool) {
+func (this *HTTPRequest) isUAMRequest() bool {
+	// stub
+	return false
+}
+
+// UAM
+func (this *HTTPRequest) doUAM() (block bool) {
+	// stub
 	return false
 }

internal/nodes/http_request_waf.go

@@ -67,8 +67,8 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
 
 	// 当前服务的独立设置
 	if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
-		blocked, breakChecking := this.checkWAFRequest(this.web.FirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, false)
-		if blocked {
+		blockedRequest, breakChecking := this.checkWAFRequest(this.web.FirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, false)
+		if blockedRequest {
 			return true
 		}
 		if breakChecking {
@@ -78,8 +78,8 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
 
 	// 公用的防火墙设置
 	if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
-		blocked, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, this.web.FirewallRef.IgnoreGlobalRules)
-		if blocked {
+		blockedRequest, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying, this.web.FirewallRef.IgnoreGlobalRules)
+		if blockedRequest {
 			return true
 		}
 		if breakChecking {
@@ -96,6 +96,8 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 		return
 	}
 
+	var isDefendMode = firewallPolicy.Mode == firewallconfigs.FirewallModeDefend
+
 	// 检查IP白名单
 	var remoteAddrs []string
 	if len(this.remoteAddr) > 0 {
@@ -122,7 +124,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 	}
 
 	// 检查IP黑名单
-	if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
+	if isDefendMode {
 		for _, ref := range inbound.AllDenyListRefs() {
 			if ref.IsOn && ref.ListId > 0 {
 				list := iplibrary.SharedIPListManager.FindList(ref.ListId)
@@ -161,19 +163,20 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 	}
 
 	// 检查地区封禁
-	if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
-		if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
-			var regionConfig = firewallPolicy.Inbound.Region
-			if regionConfig.IsNotEmpty() {
-				for _, remoteAddr := range remoteAddrs {
-					var result = iplib.LookupIP(remoteAddr)
-					if result != nil && result.IsOk() {
-						var currentURL = this.URL()
-						if regionConfig.MatchCountryURL(currentURL) {
-							// 检查国家/地区级别封禁
-							if !regionConfig.IsAllowedCountry(result.CountryId(), result.ProvinceId()) {
-								this.firewallPolicyId = firewallPolicy.Id
 
+	if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
+		var regionConfig = firewallPolicy.Inbound.Region
+		if regionConfig.IsNotEmpty() {
+			for _, remoteAddr := range remoteAddrs {
+				var result = iplib.LookupIP(remoteAddr)
+				if result != nil && result.IsOk() {
+					var currentURL = this.URL()
+					if regionConfig.MatchCountryURL(currentURL) {
+						// 检查国家/地区级别封禁
+						if !regionConfig.IsAllowedCountry(result.CountryId(), result.ProvinceId()) {
+							this.firewallPolicyId = firewallPolicy.Id
+
+							if isDefendMode {
 								var promptHTML string
 								if len(regionConfig.CountryHTML) > 0 {
 									promptHTML = regionConfig.CountryHTML
@@ -193,23 +196,27 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 
 								// 延时返回，避免攻击
 								time.Sleep(1 * time.Second)
+							}
 
-								// 停止日志
-								if !logDenying {
-									this.disableLog = true
-								} else {
-									this.tags = append(this.tags, "denyCountry")
-								}
+							// 停止日志
+							if !logDenying {
+								this.disableLog = true
+							} else {
+								this.tags = append(this.tags, "denyCountry")
+							}
 
+							if isDefendMode {
 								return true, false
 							}
 						}
+					}
 
-						if regionConfig.MatchProvinceURL(currentURL) {
-							// 检查省份封禁
-							if !regionConfig.IsAllowedProvince(result.CountryId(), result.ProvinceId()) {
-								this.firewallPolicyId = firewallPolicy.Id
+					if regionConfig.MatchProvinceURL(currentURL) {
+						// 检查省份封禁
+						if !regionConfig.IsAllowedProvince(result.CountryId(), result.ProvinceId()) {
+							this.firewallPolicyId = firewallPolicy.Id
 
+							if isDefendMode {
 								var promptHTML string
 								if len(regionConfig.ProvinceHTML) > 0 {
 									promptHTML = regionConfig.ProvinceHTML
@@ -229,14 +236,16 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 
 								// 延时返回，避免攻击
 								time.Sleep(1 * time.Second)
+							}
 
-								// 停止日志
-								if !logDenying {
-									this.disableLog = true
-								} else {
-									this.tags = append(this.tags, "denyProvince")
-								}
+							// 停止日志
+							if !logDenying {
+								this.disableLog = true
+							} else {
+								this.tags = append(this.tags, "denyProvince")
+							}
 
+							if isDefendMode {
 								return true, false
 							}
 						}
@@ -257,8 +266,11 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 		return
 	}
 
-	goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer)
-	if forceLog && logRequestBody && hasRequestBody && ruleSet != nil && ruleSet.HasAttackActions() {
+	result, err := w.MatchRequest(this, this.writer, this.web.FirewallRef.DefaultCaptchaType)
+	if result.IsAllowed && (len(result.AllowScope) == 0 || result.AllowScope == waf.AllowScopeGlobal) {
+		breakChecking = true
+	}
+	if forceLog && logRequestBody && result.HasRequestBody && result.Set != nil && result.Set.HasAttackActions() {
 		this.wafHasRequestBody = true
 	}
 	if err != nil {
@@ -268,28 +280,28 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
 		return
 	}
 
-	if ruleSet != nil {
+	if result.Set != nil {
 		if forceLog {
 			this.forceLog = true
 		}
 
-		if ruleSet.HasSpecialActions() {
+		if result.Set.HasSpecialActions() {
 			this.firewallPolicyId = firewallPolicy.Id
-			this.firewallRuleGroupId = types.Int64(ruleGroup.Id)
-			this.firewallRuleSetId = types.Int64(ruleSet.Id)
+			this.firewallRuleGroupId = types.Int64(result.Group.Id)
+			this.firewallRuleSetId = types.Int64(result.Set.Id)
 
-			if ruleSet.HasAttackActions() {
+			if result.Set.HasAttackActions() {
 				this.isAttack = true
 			}
 
 			// 添加统计
-			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
+			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, result.Set.Actions)
 		}
 
-		this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
+		this.firewallActions = append(result.Set.ActionCodes(), firewallPolicy.Mode)
 	}
 
-	return !goNext, false
+	return !result.GoNext, breakChecking
 }
 
 // call response waf
@@ -307,23 +319,26 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
 	}
 
 	if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
-		blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
-		if blocked {
+		blockedRequest, breakChecking := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
+		if blockedRequest {
 			return true
 		}
+		if breakChecking {
+			return
+		}
 	}
 
 	// 公用的防火墙设置
 	if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
-		blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
-		if blocked {
+		blockedRequest, _ := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
+		if blockedRequest {
 			return true
 		}
 	}
 	return
 }
 
-func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response, forceLog bool, logRequestBody bool, ignoreRules bool) (blocked bool) {
+func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response, forceLog bool, logRequestBody bool, ignoreRules bool) (blocked bool, breakChecking bool) {
 	if firewallPolicy == nil || !firewallPolicy.IsOn || !firewallPolicy.Outbound.IsOn || firewallPolicy.Mode == firewallconfigs.FirewallModeBypass {
 		return
 	}
@@ -338,8 +353,11 @@ func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFi
 		return
 	}
 
-	goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchResponse(this, resp, this.writer)
-	if forceLog && logRequestBody && hasRequestBody && ruleSet != nil && ruleSet.HasAttackActions() {
+	result, err := w.MatchResponse(this, resp, this.writer)
+	if result.IsAllowed && (len(result.AllowScope) == 0 || result.AllowScope == waf.AllowScopeGlobal) {
+		breakChecking = true
+	}
+	if forceLog && logRequestBody && result.HasRequestBody && result.Set != nil && result.Set.HasAttackActions() {
 		this.wafHasRequestBody = true
 	}
 	if err != nil {
@@ -349,28 +367,28 @@ func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFi
 		return
 	}
 
-	if ruleSet != nil {
+	if result.Set != nil {
 		if forceLog {
 			this.forceLog = true
 		}
 
-		if ruleSet.HasSpecialActions() {
+		if result.Set.HasSpecialActions() {
 			this.firewallPolicyId = firewallPolicy.Id
-			this.firewallRuleGroupId = types.Int64(ruleGroup.Id)
-			this.firewallRuleSetId = types.Int64(ruleSet.Id)
+			this.firewallRuleGroupId = types.Int64(result.Group.Id)
+			this.firewallRuleSetId = types.Int64(result.Set.Id)
 
-			if ruleSet.HasAttackActions() {
+			if result.Set.HasAttackActions() {
 				this.isAttack = true
 			}
 
 			// 添加统计
-			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
+			stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, result.Set.Actions)
 		}
 
-		this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
+		this.firewallActions = append(result.Set.ActionCodes(), firewallPolicy.Mode)
 	}
 
-	return !goNext
+	return !result.GoNext, breakChecking
 }
 
 // WAFRaw 原始请求
@@ -469,3 +487,10 @@ func (this *HTTPRequest) WAFMaxRequestSize() int64 {
 func (this *HTTPRequest) DisableAccessLog() {
 	this.disableLog = true
 }
+
+// DisableStat 停用统计
+func (this *HTTPRequest) DisableStat() {
+	if this.web != nil {
+		this.web.StatRef = nil
+	}
+}

internal/nodes/http_request_websocket.go

@@ -61,6 +61,9 @@ func (this *HTTPRequest) doWebsocket(requestHost string, isLastRetry bool) (shou
 		}
 	}
 
+	// 标记
+	this.isWebsocketResponse = true
+
 	// 设置指定的来源域
 	if !this.web.Websocket.RequestSameOrigin && len(this.web.Websocket.RequestOrigin) > 0 {
 		var newRequestOrigin = this.web.Websocket.RequestOrigin
@@ -77,7 +80,6 @@ func (this *HTTPRequest) doWebsocket(requestHost string, isLastRetry bool) (shou
 	}
 
 	// 连接源站
-	// TODO 增加N次错误重试，重试的时候需要尝试不同的源站
 	originConn, _, err := OriginConnect(this.origin, this.requestServerPort(), this.RawReq.RemoteAddr, requestHost)
 	if err != nil {
 		if isLastRetry {

internal/nodes/http_writer.go

@@ -11,7 +11,6 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/TeaOSLab/EdgeNode/internal/compressions"
-	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
@@ -19,7 +18,6 @@ import (
 	setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/writers"
 	_ "github.com/biessek/golang-ico"
-	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	"github.com/iwind/gowebp"
 	_ "golang.org/x/image/bmp"
@@ -34,22 +32,19 @@ import (
 	"net/textproto"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync/atomic"
 )
 
-var webpMaxBufferSize int64 = 1_000_000_000
-var webpTotalBufferSize int64 = 0
-var webpIgnoreURLSet = setutils.NewFixedSet(131072)
+var webPThreads int32
+var webPMaxThreads int32 = 1
+var webPIgnoreURLSet = setutils.NewFixedSet(131072)
 
 func init() {
-	if !teaconst.IsMain {
-		return
-	}
-
-	var systemMemory = utils.SystemMemoryGB() / 8
-	if systemMemory > 0 {
-		webpMaxBufferSize = int64(systemMemory) << 30
+	webPMaxThreads = int32(runtime.NumCPU() / 4)
+	if webPMaxThreads < 1 {
+		webPMaxThreads = 1
 	}
 }
 
@@ -80,6 +75,7 @@ type HTTPWriter struct {
 	// WebP
 	webpIsEncoding        bool
 	webpOriginContentType string
+	webpQuality           int
 
 	// Compression
 	compressionConfig      *serverconfigs.HTTPCompressionConfig
@@ -343,7 +339,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 
 	cacheWriter, err := storage.OpenWriter(cacheKey, expiresAt, this.StatusCode(), this.calculateHeaderLength(), totalSize, cacheRef.MaxSizeBytes(), this.isPartial)
 	if err != nil {
-		if err == caches.ErrEntityTooLarge && addStatusHeader {
+		if errors.Is(err, caches.ErrEntityTooLarge) && addStatusHeader {
 			this.Header().Set("X-Cache", "BYPASS, entity too large")
 		}
 
@@ -483,8 +479,8 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 					contentTypeWritten = true
 				}
 
-				err := cacheWriter.WriteAt(start, data)
-				if err != nil {
+				writeErr := cacheWriter.WriteAt(start, data)
+				if writeErr != nil {
 					hasError = true
 					this.cacheIsFinished = false
 				}
@@ -497,7 +493,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
 		return
 	}
 
-	var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter)
+	var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter, false)
 	resp.Body = cacheReader
 	this.rawReader = cacheReader
 
@@ -531,6 +527,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
 	if policy.RequireCache && this.req.cacheRef == nil {
 		return
 	}
+	this.webpQuality = policy.Quality
 
 	// 限制最小和最大尺寸
 	// TODO 需要将reader修改为LimitReader
@@ -550,7 +547,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
 		this.req.web.WebP.MatchResponse(contentType, size, filepath.Ext(this.req.Path()), this.req.Format) &&
 		this.req.web.WebP.MatchAccept(this.req.requestHeader("Accept")) {
 		// 检查是否已经因为尺寸过大而忽略
-		if webpIgnoreURLSet.Has(this.req.URL()) {
+		if webPIgnoreURLSet.Has(this.req.URL()) {
 			return
 		}
 
@@ -560,24 +557,24 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
 			return
 		}
 
-		// 检查内存
-		if atomic.LoadInt64(&webpTotalBufferSize) >= webpMaxBufferSize {
+		// 检查当前是否正在转换
+		if atomic.LoadInt32(&webPThreads) >= webPMaxThreads {
 			return
 		}
 
 		var contentEncoding = this.GetHeader("Content-Encoding")
-		switch contentEncoding {
-		case "gzip", "deflate", "br", "zstd":
-			reader, err := compressions.NewReader(resp.Body, contentEncoding)
-			if err != nil {
+		if len(contentEncoding) > 0 {
+			if compressions.SupportEncoding(contentEncoding) {
+				reader, err := compressions.NewReader(resp.Body, contentEncoding)
+				if err != nil {
+					return
+				}
+				this.Header().Del("Content-Encoding")
+				this.Header().Del("Content-Length")
+				this.rawReader = reader
+			} else {
 				return
 			}
-			this.Header().Del("Content-Encoding")
-			this.Header().Del("Content-Length")
-			this.rawReader = reader
-		case "": // 空
-		default:
-			return
 		}
 
 		this.webpOriginContentType = contentType
@@ -605,7 +602,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
 	var contentEncoding = this.GetHeader("Content-Encoding")
 
 	if this.compressionConfig == nil || !this.compressionConfig.IsOn {
-		if lists.ContainsString([]string{"gzip", "deflate", "br", "zstd"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
+		if compressions.SupportEncoding(contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
 			reader, err := compressions.NewReader(resp.Body, contentEncoding)
 			if err != nil {
 				return
@@ -622,12 +619,12 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
 		return
 	}
 
-	if this.compressionConfig.Level <= 0 {
+	if this.compressionConfig.Level < 0 {
 		return
 	}
 
 	// 如果已经有编码则不处理
-	if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !lists.ContainsString([]string{"gzip", "deflate", "br", "zstd"}, contentEncoding)) {
+	if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !compressions.SupportEncoding(contentEncoding)) {
 		return
 	}
 
@@ -809,6 +806,8 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) {
 	}
 	n, err = this.writer.Write(data)
 
+	this.checkPlanBandwidth(n)
+
 	return
 }
 
@@ -969,6 +968,7 @@ func (this *HTTPWriter) Close() {
 func (this *HTTPWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
 	hijack, ok := this.rawWriter.(http.Hijacker)
 	if ok {
+		this.req.isHijacked = true
 		return hijack.Hijack()
 	}
 	return
@@ -1020,6 +1020,11 @@ func (this *HTTPWriter) calculateStaleLife() int {
 func (this *HTTPWriter) finishWebP() {
 	// 处理WebP
 	if this.webpIsEncoding {
+		atomic.AddInt32(&webPThreads, 1)
+		defer func() {
+			atomic.AddInt32(&webPThreads, -1)
+		}()
+
 		var webpCacheWriter caches.Writer
 
 		// 准备WebP Cache
@@ -1080,7 +1085,7 @@ func (this *HTTPWriter) finishWebP() {
 		if isGif {
 			gifImage, err = gif.DecodeAll(reader)
 			if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
-				webpIgnoreURLSet.Push(this.req.URL())
+				webPIgnoreURLSet.Push(this.req.URL())
 				return
 			}
 		} else {
@@ -1088,7 +1093,7 @@ func (this *HTTPWriter) finishWebP() {
 			if imageData != nil {
 				var bound = imageData.Bounds()
 				if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
-					webpIgnoreURLSet.Push(this.req.URL())
+					webPIgnoreURLSet.Push(this.req.URL())
 					return
 				}
 			}
@@ -1096,19 +1101,21 @@ func (this *HTTPWriter) finishWebP() {
 
 		if err != nil {
 			// 发生了错误终止处理
-			webpIgnoreURLSet.Push(this.req.URL())
+			webPIgnoreURLSet.Push(this.req.URL())
 			return
 		}
 
-		var totalBytes = reader.TotalBytes()
-		atomic.AddInt64(&webpTotalBufferSize, totalBytes)
-		defer func() {
-			atomic.AddInt64(&webpTotalBufferSize, -totalBytes)
-		}()
-
-		var f = types.Float32(this.req.web.WebP.Quality)
-		if f > 100 {
-			f = 100
+		var f = types.Float32(this.webpQuality)
+		if f <= 0 || f > 100 {
+			if this.size > (8<<20) || this.size <= 0 {
+				f = 30
+			} else if this.size > (1 << 20) {
+				f = 50
+			} else if this.size > (128 << 10) {
+				f = 60
+			} else {
+				f = 75
+			}
 		}
 
 		if imageData != nil {

internal/nodes/http_writer_ext.go

@@ -11,3 +11,7 @@ import (
 func (this *HTTPWriter) canSendfile() (*os.File, bool) {
 	return nil, false
 }
+
+func (this *HTTPWriter) checkPlanBandwidth(n int) {
+	// stub
+}

internal/nodes/listener_base.go

@@ -46,7 +46,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
 				}
 			}
 
-			tlsPolicy, _, err := this.matchSSL(this.helloServerName(clientInfo))
+			tlsPolicy, _, err := this.matchSSL(this.helloServerNames(clientInfo))
 			if err != nil {
 				return nil, err
 			}
@@ -69,7 +69,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
 				}
 			}
 
-			tlsPolicy, cert, err := this.matchSSL(this.helloServerName(clientInfo))
+			tlsPolicy, cert, err := this.matchSSL(this.helloServerNames(clientInfo))
 			if err != nil {
 				return nil, err
 			}
@@ -85,7 +85,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
 }
 
 // 根据域名匹配证书
-func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
+func (this *BaseListener) matchSSL(domains []string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
 	var group = this.Group
 
 	if group == nil {
@@ -99,7 +99,7 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 
 	// 如果域名为空，则取第一个
 	// 通常域名为空是因为是直接通过IP访问的
-	if len(domain) == 0 {
+	if len(domains) == 0 {
 		if group.IsHTTPS() && globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
 			return nil, nil, errors.New("no tls server name matched")
 		}
@@ -116,9 +116,25 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 		}
 		return nil, nil, errors.New("no tls server name found")
 	}
+	var firstDomain = domains[0]
 
 	// 通过网站域名配置匹配
-	server, _ := this.findNamedServer(domain)
+	var server *serverconfigs.ServerConfig
+	var matchedDomain string
+	for _, domain := range domains {
+		server, _ = this.findNamedServer(domain, true)
+		if server != nil {
+			matchedDomain = domain
+			break
+		}
+	}
+	if server == nil {
+		server, _ = this.findNamedServer(firstDomain, false)
+		if server != nil {
+			matchedDomain = firstDomain
+		}
+	}
+
 	if server == nil {
 		// 找不到或者此时的服务没有配置证书，需要搜索所有的Server，通过SSL证书内容中的DNSName匹配
 		// 此功能仅为了兼容以往版本（v1.0.4），不应该作为常态启用
@@ -127,14 +143,14 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 				if searchingServer.SSLPolicy() == nil || !searchingServer.SSLPolicy().IsOn {
 					continue
 				}
-				cert, ok := searchingServer.SSLPolicy().MatchDomain(domain)
+				cert, ok := searchingServer.SSLPolicy().MatchDomain(firstDomain)
 				if ok {
 					return searchingServer.SSLPolicy(), cert, nil
 				}
 			}
 		}
 
-		return nil, nil, errors.New("no server found for '" + domain + "'")
+		return nil, nil, errors.New("no server found for '" + firstDomain + "'")
 	}
 	if server.SSLPolicy() == nil || !server.SSLPolicy().IsOn {
 		// 找不到或者此时的服务没有配置证书，需要搜索所有的Server，通过SSL证书内容中的DNSName匹配
@@ -144,32 +160,32 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
 				if searchingServer.SSLPolicy() == nil || !searchingServer.SSLPolicy().IsOn {
 					continue
 				}
-				cert, ok := searchingServer.SSLPolicy().MatchDomain(domain)
+				cert, ok := searchingServer.SSLPolicy().MatchDomain(matchedDomain)
 				if ok {
 					return searchingServer.SSLPolicy(), cert, nil
 				}
 			}
 		}
 
-		return nil, nil, errors.New("no cert found for '" + domain + "'")
+		return nil, nil, errors.New("no cert found for '" + matchedDomain + "'")
 	}
 
 	// 证书是否匹配
 	var sslConfig = server.SSLPolicy()
-	cert, ok := sslConfig.MatchDomain(domain)
+	cert, ok := sslConfig.MatchDomain(matchedDomain)
 	if ok {
 		return sslConfig, cert, nil
 	}
 
 	if len(sslConfig.Certs) == 0 {
-		remotelogs.ServerError(server.Id, "BASE_LISTENER", "no ssl certs found for '"+domain+"', server id: "+types.String(server.Id), "", nil)
+		remotelogs.ServerError(server.Id, "BASE_LISTENER", "no ssl certs found for '"+matchedDomain+"', server id: "+types.String(server.Id), "", nil)
 	}
 
 	return sslConfig, sslConfig.FirstCert(), nil
 }
 
 // 根据域名来查找匹配的域名
-func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconfigs.ServerConfig, serverName string) {
+func (this *BaseListener) findNamedServer(name string, exactly bool) (serverConfig *serverconfigs.ServerConfig, serverName string) {
 	serverConfig, serverName = this.findNamedServerMatched(name)
 	if serverConfig != nil {
 		return
@@ -194,18 +210,22 @@ func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconf
 		}
 	}
 
-	if matchDomainStrictly && !configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) && (!globalServerConfig.HTTPAll.AllowNodeIP || !utils.IsWildIP(name)) {
+	if matchDomainStrictly && !configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) && (!globalServerConfig.HTTPAll.AllowNodeIP || (!utils.IsWildIP(name) || globalServerConfig.HTTPAll.NodeIPShowPage)) {
 		return
 	}
 
-	// 如果没有找到，则匹配到第一个
-	var group = this.Group
-	var currentServers = group.Servers()
-	var countServers = len(currentServers)
-	if countServers == 0 {
-		return nil, ""
+	if !exactly {
+		// 如果没有找到，则匹配到第一个
+		var group = this.Group
+		var currentServers = group.Servers()
+		var countServers = len(currentServers)
+		if countServers == 0 {
+			return nil, ""
+		}
+		return currentServers[0], name
 	}
-	return currentServers[0], name
+
+	return
 }
 
 // 严格查找域名
@@ -234,16 +254,23 @@ func (this *BaseListener) findNamedServerMatched(name string) (serverConfig *ser
 }
 
 // 从Hello信息中获取服务名称
-func (this *BaseListener) helloServerName(clientInfo *tls.ClientHelloInfo) string {
-	var serverName = clientInfo.ServerName
-	if len(serverName) == 0 && clientInfo.Conn != nil {
+func (this *BaseListener) helloServerNames(clientInfo *tls.ClientHelloInfo) (serverNames []string) {
+	if len(clientInfo.ServerName) != 0 {
+		serverNames = append(serverNames, clientInfo.ServerName)
+		return
+	}
+
+	if clientInfo.Conn != nil {
 		var localAddr = clientInfo.Conn.LocalAddr()
 		if localAddr != nil {
 			tcpAddr, ok := localAddr.(*net.TCPAddr)
 			if ok {
-				serverName = tcpAddr.IP.String()
+				serverNames = append(serverNames, tcpAddr.IP.String())
 			}
 		}
 	}
-	return serverName
+
+	serverNames = append(serverNames, sharedNodeConfig.IPAddresses...)
+
+	return
 }

internal/nodes/listener_http.go

@@ -3,6 +3,7 @@ package nodes
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/iwind/TeaGo/Tea"
 	"io"
@@ -52,6 +53,8 @@ func (this *HTTPListener) Serve() error {
 				atomic.AddInt64(&this.countActiveConnections, 1)
 			case http.StateClosed:
 				atomic.AddInt64(&this.countActiveConnections, -1)
+			default:
+				// do nothing
 			}
 		},
 		ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
@@ -74,7 +77,7 @@ func (this *HTTPListener) Serve() error {
 	// HTTP协议
 	if this.isHTTP {
 		err := this.httpServer.Serve(this.Listener)
-		if err != nil && err != http.ErrServerClosed {
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return err
 		}
 	}
@@ -84,7 +87,7 @@ func (this *HTTPListener) Serve() error {
 		this.httpServer.TLSConfig = this.buildTLSConfig()
 
 		err := this.httpServer.ServeTLS(this.Listener, "", "")
-		if err != nil && err != http.ErrServerClosed {
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
 			return err
 		}
 	}
@@ -105,17 +108,30 @@ func (this *HTTPListener) Reload(group *serverconfigs.ServerAddressGroup) {
 	this.Reset()
 }
 
-// ServerHTTP 处理HTTP请求
+// ServeHTTPWithAddr 处理HTTP请求
 func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.Request) {
+	this.ServeHTTPWithAddr(rawWriter, rawReq, this.addr)
+}
+
+// ServeHTTPWithAddr 处理HTTP请求并指定服务地址
+func (this *HTTPListener) ServeHTTPWithAddr(rawWriter http.ResponseWriter, rawReq *http.Request, serverAddr string) {
+	if len(rawReq.Host) > 253 {
+		http.Error(rawWriter, "Host too long.", http.StatusBadRequest)
+		time.Sleep(1 * time.Second) // make connection slow down
+		return
+	}
+
 	var globalServerConfig = sharedNodeConfig.GlobalServerConfig
 	if globalServerConfig != nil && !globalServerConfig.HTTPAll.SupportsLowVersionHTTP && (rawReq.ProtoMajor < 1 /** 0.x **/ || (rawReq.ProtoMajor == 1 && rawReq.ProtoMinor == 0 /** 1.0 **/)) {
 		http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusBadRequest)
+		time.Sleep(1 * time.Second) // make connection slow down
 		return
 	}
 
 	// 不支持Connect
 	if rawReq.Method == http.MethodConnect {
 		http.Error(rawWriter, "Method Not Allowed", http.StatusMethodNotAllowed)
+		time.Sleep(1 * time.Second) // make connection slow down
 		return
 	}
 
@@ -154,7 +170,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 		domain = reqHost
 	}
 
-	server, serverName := this.findNamedServer(domain)
+	server, serverName := this.findNamedServer(domain, false)
 	if server == nil {
 		if server == nil {
 			// 增加默认的一个服务
@@ -167,10 +183,12 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 	}
 
 	// 绑定连接
+	var clientConn ClientConnInterface
 	if server != nil && server.Id > 0 {
 		var requestConn = rawReq.Context().Value(HTTPConnContextKey)
 		if requestConn != nil {
-			clientConn, ok := requestConn.(ClientConnInterface)
+			var ok bool
+			clientConn, ok = requestConn.(ClientConnInterface)
 			if ok {
 				var goNext = clientConn.SetServerId(server.Id)
 				if !goNext {
@@ -203,7 +221,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 		ReqServer:  server,
 		ReqHost:    reqHost,
 		ServerName: serverName,
-		ServerAddr: this.addr,
+		ServerAddr: serverAddr,
 		IsHTTP:     this.isHTTP,
 		IsHTTPS:    this.isHTTPS,
 		IsHTTP3:    this.isHTTP3,
@@ -211,6 +229,14 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
 		nodeConfig: sharedNodeConfig,
 	}
 	req.Do()
+
+	// fix hijacked connection state
+	if req.isHijacked && clientConn != nil && this.httpServer.ConnState != nil {
+		netConn, ok := clientConn.(net.Conn)
+		if ok {
+			this.httpServer.ConnState(netConn, http.StateClosed)
+		}
+	}
 }
 
 // 检查host是否为IP

internal/nodes/listener_tcp.go

@@ -47,9 +47,13 @@ func (this *TCPListener) Serve() error {
 		atomic.AddInt64(&this.countActiveConnections, 1)
 
 		go func(conn net.Conn) {
-			err = this.handleConn(conn)
+			var server = this.Group.FirstServer()
+			if server == nil {
+				return
+			}
+			err = this.handleConn(server, conn)
 			if err != nil {
-				remotelogs.Error("TCP_LISTENER", err.Error())
+				remotelogs.ServerError(server.Id, "TCP_LISTENER", err.Error(), "", nil)
 			}
 			atomic.AddInt64(&this.countActiveConnections, -1)
 		}(conn)
@@ -63,8 +67,7 @@ func (this *TCPListener) Reload(group *serverconfigs.ServerAddressGroup) {
 	this.Reset()
 }
 
-func (this *TCPListener) handleConn(conn net.Conn) error {
-	var server = this.Group.FirstServer()
+func (this *TCPListener) handleConn(server *serverconfigs.ServerConfig, conn net.Conn) error {
 	if server == nil {
 		return errors.New("no server available")
 	}
@@ -132,14 +135,14 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
 		serverName = tlsConn.ConnectionState().ServerName
 		if len(serverName) > 0 {
 			// 统计
-			stats.SharedTrafficStatManager.Add(server.UserId, server.Id, serverName, 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
+			stats.SharedTrafficStatManager.Add(server.UserId, server.Id, serverName, 0, 0, 1, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
 			recordStat = true
 		}
 	}
 
 	// 统计
 	if !recordStat {
-		stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
+		stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
 	}
 
 	originConn, err := this.connectOrigin(server.Id, serverName, server.ReverseProxy, conn.RemoteAddr().String())
@@ -194,7 +197,7 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
 
 				// 记录流量
 				if server != nil {
-					stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
+					stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
 				}
 			}
 			if err != nil {

internal/nodes/listener_udp.go

@@ -370,7 +370,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyListener
 
 	// 统计
 	if server != nil {
-		stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
+		stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
 	}
 
 	// 处理ControlMessage
@@ -401,7 +401,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyListener
 				// 记录流量和带宽
 				if server != nil {
 					// 流量
-					stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
+					stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
 
 					// 带宽
 					var userPlanId int64

internal/nodes/node.go

@@ -146,12 +146,12 @@ func (this *Node) Start() {
 	remotelogs.Println("NODE", "init config ...")
 	err = this.syncConfig(0)
 	if err != nil {
-		_, err := nodeconfigs.SharedNodeConfig()
+		_, err = nodeconfigs.SharedNodeConfig()
 		if err != nil {
 			// 无本地数据时，会尝试多次读取
 			tryTimes := 0
 			for {
-				err := this.syncConfig(0)
+				err = this.syncConfig(0)
 				if err != nil {
 					tryTimes++
 
@@ -777,9 +777,23 @@ func (this *Node) listenSock() error {
 					_ = cmd.ReplyOk()
 				}
 			case "gc":
+				var before = time.Now()
 				runtime.GC()
 				debug.FreeOSMemory()
-				_ = cmd.ReplyOk()
+
+				var costSeconds = time.Since(before).Seconds()
+				var gcStats = &debug.GCStats{}
+				debug.ReadGCStats(gcStats)
+				var pauseMS float64
+				if len(gcStats.Pause) > 0 {
+					pauseMS = gcStats.Pause[0].Seconds() * 1000
+				}
+				_ = cmd.Reply(&gosock.Command{
+					Params: map[string]any{
+						"pauseMS": pauseMS,
+						"costMS":  costSeconds * 1000,
+					},
+				})
 			case "reload":
 				err := this.syncConfig(0)
 				if err != nil {
@@ -1039,7 +1053,7 @@ func (this *Node) reloadServer() {
 		for serverId, serverConfig := range updatingServerMap {
 			if serverConfig != nil {
 				if countUpdatingServers < maxPrintServers {
-					remotelogs.Debug("NODE", "load server '"+types.String(serverId)+"'")
+					remotelogs.Debug("NODE", "reload server '"+types.String(serverId)+"'")
 				}
 				newNodeConfig.AddServer(serverConfig)
 			} else {

internal/nodes/node_status_executor.go

@@ -223,6 +223,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 	// 当前TeaWeb所在的fs
 	var rootFS = ""
 	var rootTotal = uint64(0)
+	var totalUsed = uint64(0)
 	if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) {
 		for _, p := range partitions {
 			if p.Mountpoint == "/" {
@@ -230,6 +231,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 				usage, _ := disk.Usage(p.Mountpoint)
 				if usage != nil {
 					rootTotal = usage.Total
+					totalUsed = usage.Used
 				}
 				break
 			}
@@ -237,7 +239,6 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 	}
 
 	var total = rootTotal
-	var totalUsage = uint64(0)
 	var maxUsage = float64(0)
 	for _, partition := range partitions {
 		if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") {
@@ -256,16 +257,16 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
 
 		if partition.Mountpoint != "/" && (usage.Total != rootTotal || total == 0) {
 			total += usage.Total
-		}
-		totalUsage += usage.Used
-		if usage.UsedPercent >= maxUsage {
-			maxUsage = usage.UsedPercent
-			status.DiskMaxUsagePartition = partition.Mountpoint
+			totalUsed += usage.Used
+			if usage.UsedPercent >= maxUsage {
+				maxUsage = usage.UsedPercent
+				status.DiskMaxUsagePartition = partition.Mountpoint
+			}
 		}
 	}
 	status.DiskTotal = total
 	if total > 0 {
-		status.DiskUsage = float64(totalUsage) / float64(total)
+		status.DiskUsage = float64(totalUsed) / float64(total)
 	}
 	status.DiskMaxUsage = maxUsage / 100
 

internal/nodes/node_tasks.go

@@ -17,6 +17,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
 	"github.com/TeaOSLab/EdgeNode/internal/trackers"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/waf"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/maps"
@@ -97,6 +98,12 @@ func (this *Node) execTask(rpcClient *rpc.RPCClient, task *pb.NodeTask) error {
 		err = this.notifyPlusChange()
 	case "toaChanged":
 		err = this.execTOAChangedTask()
+	case "networkSecurityPolicyChanged":
+		err = this.execNetworkSecurityPolicyChangedTask(rpcClient)
+	case "webPPolicyChanged":
+		err = this.execWebPPolicyChangedTask(rpcClient)
+	case "planChanged":
+		err = this.execPlanChangedTask(rpcClient)
 	default:
 		// 特殊任务
 		if strings.HasPrefix(task.Type, "ipListDeleted") { // 删除IP名单
@@ -296,7 +303,7 @@ func (this *Node) execUpdatingServersTask(rpcClient *rpc.RPCClient) error {
 
 // 删除IP名单
 func (this *Node) execDeleteIPList(taskType string) error {
-	optionsString, ok := strings.CutPrefix(taskType, "ipListDeleted@")
+	optionsString, ok := utils.CutPrefix(taskType, "ipListDeleted@")
 	if !ok {
 		return errors.New("invalid task type '" + taskType + "'")
 	}
@@ -322,6 +329,34 @@ func (this *Node) execDeleteIPList(taskType string) error {
 	return nil
 }
 
+// WebP策略变更
+func (this *Node) execWebPPolicyChangedTask(rpcClient *rpc.RPCClient) error {
+	remotelogs.Println("NODE", "updating webp policies ...")
+	resp, err := rpcClient.NodeRPC.FindNodeWebPPolicies(rpcClient.Context(), &pb.FindNodeWebPPoliciesRequest{})
+	if err != nil {
+		return err
+	}
+	var webPPolicyMap = map[int64]*nodeconfigs.WebPImagePolicy{}
+	for _, policy := range resp.WebPPolicies {
+		if len(policy.WebPPolicyJSON) > 0 {
+			var webPPolicy = nodeconfigs.NewWebPImagePolicy()
+			err = json.Unmarshal(policy.WebPPolicyJSON, webPPolicy)
+			if err != nil {
+				remotelogs.Error("NODE", "decode webp policy failed: "+err.Error())
+				continue
+			}
+			err = webPPolicy.Init()
+			if err != nil {
+				remotelogs.Error("NODE", "initialize webp policy failed: "+err.Error())
+				continue
+			}
+			webPPolicyMap[policy.NodeClusterId] = webPPolicy
+		}
+	}
+	sharedNodeConfig.UpdateWebPImagePolicies(webPPolicyMap)
+	return nil
+}
+
 // 标记任务完成
 func (this *Node) finishTask(taskId int64, taskVersion int64, taskErr error) (success bool) {
 	if taskId <= 0 {

internal/nodes/node_tasks_ext.go

@@ -29,3 +29,12 @@ func (this *Node) execHTTPPagesPolicyChangedTask(rpcClient *rpc.RPCClient) error
 	// stub
 	return nil
 }
+
+func (this *Node) execNetworkSecurityPolicyChangedTask(rpcClient *rpc.RPCClient) error {
+	// stub
+	return nil
+}
+
+func (this *Node) execPlanChangedTask(rpcClient *rpc.RPCClient) error {
+	return nil
+}

internal/nodes/node_test.go

@@ -1,17 +1,26 @@
 package nodes
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
 )
 
 func TestNode_Start(t *testing.T) {
-	node := NewNode()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var node = NewNode()
 	node.Start()
 }
 
 func TestNode_Test(t *testing.T) {
-	node := NewNode()
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
+	var node = NewNode()
 	err := node.Test()
 	if err != nil {
 		t.Fatal(err)

internal/nodes/upgrade_manager_test.go

@@ -3,11 +3,16 @@
 package nodes
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
 )
 
 func TestUpgradeManager_install(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	err := NewUpgradeManager().install()
 	if err != nil {
 		t.Fatal(err)

internal/re/rune_tree.go

@@ -4,7 +4,7 @@ package re
 
 type RuneMap map[rune]*RuneTree
 
-func (this *RuneMap) Lookup(s string, caseInsensitive bool) bool {
+func (this RuneMap) Lookup(s string, caseInsensitive bool) bool {
 	return this.lookup([]rune(s), caseInsensitive, 0)
 }
 

internal/rpc/rpc_client.go

@@ -18,6 +18,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/encoding/gzip"
+	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	"net/url"
 	"sync"
@@ -54,6 +55,7 @@ type RPCClient struct {
 	ClientAgentIPRPC       pb.ClientAgentIPServiceClient
 	AuthorityKeyRPC        pb.AuthorityKeyServiceClient
 	UpdatingServerListRPC  pb.UpdatingServerListServiceClient
+	PlanRPC                pb.PlanServiceClient
 }
 
 func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
@@ -90,6 +92,7 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
 	client.ClientAgentIPRPC = pb.NewClientAgentIPServiceClient(client)
 	client.AuthorityKeyRPC = pb.NewAuthorityKeyServiceClient(client)
 	client.UpdatingServerListRPC = pb.NewUpdatingServerListServiceClient(client)
+	client.PlanRPC = pb.NewPlanServiceClient(client)
 
 	err := client.init()
 	if err != nil {
@@ -240,12 +243,15 @@ func (this *RPCClient) init() error {
 			grpc.MaxCallSendMsgSize(512<<20),
 			grpc.UseCompressor(gzip.Name),
 		)
+		var keepaliveParams = grpc.WithKeepaliveParams(keepalive.ClientParameters{
+			Time: 30 * time.Second,
+		})
 		if u.Scheme == "http" {
-			conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(insecure.NewCredentials()), callOptions)
+			conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(insecure.NewCredentials()), callOptions, keepaliveParams)
 		} else if u.Scheme == "https" {
 			conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
 				InsecureSkipVerify: true,
-			})), callOptions)
+			})), callOptions, keepaliveParams)
 		} else {
 			return errors.New("parse endpoint failed: invalid scheme '" + u.Scheme + "'")
 		}

internal/rpc/rpc_test.go

@@ -5,6 +5,7 @@ package rpc_test
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"sync"
@@ -13,6 +14,10 @@ import (
 )
 
 func TestRPCConcurrentCall(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	rpcClient, err := rpc.SharedRPC()
 	if err != nil {
 		t.Fatal(err)
@@ -43,6 +48,10 @@ func TestRPCConcurrentCall(t *testing.T) {
 }
 
 func TestRPC_Retry(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	rpcClient, err := rpc.SharedRPC()
 	if err != nil {
 		t.Fatal(err)

internal/stats/bandwidth_stat_manager.go

@@ -57,12 +57,13 @@ type BandwidthStat struct {
 	MaxBytes         int64 `json:"maxBytes"`
 	TotalBytes       int64 `json:"totalBytes"`
 
-	CachedBytes         int64 `json:"cachedBytes"`
-	AttackBytes         int64 `json:"attackBytes"`
-	CountRequests       int64 `json:"countRequests"`
-	CountCachedRequests int64 `json:"countCachedRequests"`
-	CountAttackRequests int64 `json:"countAttackRequests"`
-	UserPlanId          int64 `json:"userPlanId"`
+	CachedBytes               int64 `json:"cachedBytes"`
+	AttackBytes               int64 `json:"attackBytes"`
+	CountRequests             int64 `json:"countRequests"`
+	CountCachedRequests       int64 `json:"countCachedRequests"`
+	CountAttackRequests       int64 `json:"countAttackRequests"`
+	CountWebsocketConnections int64 `json:"countWebsocketConnections"`
+	UserPlanId                int64 `json:"userPlanId"`
 }
 
 // BandwidthStatManager 服务带宽统计
@@ -142,20 +143,21 @@ func (this *BandwidthStatManager) Loop() error {
 			}
 
 			pbStats = append(pbStats, &pb.ServerBandwidthStat{
-				Id:                  0,
-				UserId:              stat.UserId,
-				ServerId:            stat.ServerId,
-				Day:                 stat.Day,
-				TimeAt:              stat.TimeAt,
-				Bytes:               stat.MaxBytes / bandwidthTimestampDelim,
-				TotalBytes:          stat.TotalBytes,
-				CachedBytes:         stat.CachedBytes,
-				AttackBytes:         stat.AttackBytes,
-				CountRequests:       stat.CountRequests,
-				CountCachedRequests: stat.CountCachedRequests,
-				CountAttackRequests: stat.CountAttackRequests,
-				UserPlanId:          stat.UserPlanId,
-				NodeRegionId:        regionId,
+				Id:                        0,
+				UserId:                    stat.UserId,
+				ServerId:                  stat.ServerId,
+				Day:                       stat.Day,
+				TimeAt:                    stat.TimeAt,
+				Bytes:                     stat.MaxBytes / bandwidthTimestampDelim,
+				TotalBytes:                stat.TotalBytes,
+				CachedBytes:               stat.CachedBytes,
+				AttackBytes:               stat.AttackBytes,
+				CountRequests:             stat.CountRequests,
+				CountCachedRequests:       stat.CountCachedRequests,
+				CountAttackRequests:       stat.CountAttackRequests,
+				CountWebsocketConnections: stat.CountWebsocketConnections,
+				UserPlanId:                stat.UserPlanId,
+				NodeRegionId:              regionId,
 			})
 			delete(this.m, key)
 		}
@@ -231,7 +233,7 @@ func (this *BandwidthStatManager) AddBandwidth(userId int64, userPlanId int64, s
 }
 
 // AddTraffic 添加请求数据
-func (this *BandwidthStatManager) AddTraffic(serverId int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64) {
+func (this *BandwidthStatManager) AddTraffic(serverId int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, countWebsocketConnections int64) {
 	var now = fasttime.Now()
 	var day = now.Ymd()
 	var timeAt = now.Round5Hi()
@@ -245,6 +247,7 @@ func (this *BandwidthStatManager) AddTraffic(serverId int64, cachedBytes int64,
 		stat.CountCachedRequests += countCachedRequests
 		stat.CountAttackRequests += countAttacks
 		stat.AttackBytes += attackBytes
+		stat.CountWebsocketConnections += countWebsocketConnections
 	}
 	this.locker.Unlock()
 }

internal/stats/bandwidth_stat_manager_test.go

@@ -53,19 +53,20 @@ func BenchmarkBandwidthStatManager_Slice(b *testing.B) {
 		for j := 0; j < 100; j++ {
 			var stat = &stats.BandwidthStat{}
 			pbStats = append(pbStats, &pb.ServerBandwidthStat{
-				Id:                  0,
-				UserId:              stat.UserId,
-				ServerId:            stat.ServerId,
-				Day:                 stat.Day,
-				TimeAt:              stat.TimeAt,
-				Bytes:               stat.MaxBytes / 2,
-				TotalBytes:          stat.TotalBytes,
-				CachedBytes:         stat.CachedBytes,
-				AttackBytes:         stat.AttackBytes,
-				CountRequests:       stat.CountRequests,
-				CountCachedRequests: stat.CountCachedRequests,
-				CountAttackRequests: stat.CountAttackRequests,
-				NodeRegionId:        1,
+				Id:                        0,
+				UserId:                    stat.UserId,
+				ServerId:                  stat.ServerId,
+				Day:                       stat.Day,
+				TimeAt:                    stat.TimeAt,
+				Bytes:                     stat.MaxBytes / 2,
+				TotalBytes:                stat.TotalBytes,
+				CachedBytes:               stat.CachedBytes,
+				AttackBytes:               stat.AttackBytes,
+				CountRequests:             stat.CountRequests,
+				CountCachedRequests:       stat.CountCachedRequests,
+				CountAttackRequests:       stat.CountAttackRequests,
+				CountWebsocketConnections: stat.CountWebsocketConnections,
+				NodeRegionId:              1,
 			})
 		}
 		_ = pbStats

internal/stats/traffic_stat_manager.go

@@ -106,13 +106,13 @@ func (this *TrafficStatManager) Start() {
 }
 
 // Add 添加流量
-func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, checkingTrafficLimit bool, planId int64) {
+func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, countWebsocketConnections int64, checkingTrafficLimit bool, planId int64) {
 	if serverId == 0 {
 		return
 	}
 
 	// 添加到带宽
-	SharedBandwidthStatManager.AddTraffic(serverId, cachedBytes, countRequests, countCachedRequests, countAttacks, attackBytes)
+	SharedBandwidthStatManager.AddTraffic(serverId, cachedBytes, countRequests, countCachedRequests, countAttacks, attackBytes, countWebsocketConnections)
 
 	if bytes == 0 && countRequests == 0 {
 		return
@@ -142,24 +142,26 @@ func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string,
 	item.PlanId = planId
 
 	// 单个域名流量
-	var domainKey = types.String(timestamp) + "@" + domain
-	serverDomainMap, ok := this.domainsMap[serverId]
-	if !ok {
-		serverDomainMap = map[string]*TrafficItem{}
-		this.domainsMap[serverId] = serverDomainMap
-	}
+	if len(domain) < 128 {
+		var domainKey = types.String(timestamp) + "@" + domain
+		serverDomainMap, ok := this.domainsMap[serverId]
+		if !ok {
+			serverDomainMap = map[string]*TrafficItem{}
+			this.domainsMap[serverId] = serverDomainMap
+		}
 
-	domainItem, ok := serverDomainMap[domainKey]
-	if !ok {
-		domainItem = &TrafficItem{}
-		serverDomainMap[domainKey] = domainItem
+		domainItem, ok := serverDomainMap[domainKey]
+		if !ok {
+			domainItem = &TrafficItem{}
+			serverDomainMap[domainKey] = domainItem
+		}
+		domainItem.Bytes += bytes
+		domainItem.CachedBytes += cachedBytes
+		domainItem.CountRequests += countRequests
+		domainItem.CountCachedRequests += countCachedRequests
+		domainItem.CountAttackRequests += countAttacks
+		domainItem.AttackBytes += attackBytes
 	}
-	domainItem.Bytes += bytes
-	domainItem.CachedBytes += cachedBytes
-	domainItem.CountRequests += countRequests
-	domainItem.CountCachedRequests += countCachedRequests
-	domainItem.CountAttackRequests += countAttacks
-	domainItem.AttackBytes += attackBytes
 
 	this.locker.Unlock()
 }

internal/stats/traffic_stat_manager_test.go

@@ -11,7 +11,7 @@ import (
 func TestTrafficStatManager_Add(t *testing.T) {
 	manager := NewTrafficStatManager()
 	for i := 0; i < 100; i++ {
-		manager.Add(1, 1, "goedge.cn", 1, 0, 0, 0, 0, 0, false, 0)
+		manager.Add(1, 1, "goedge.cn", 1, 0, 0, 0, 0, 0, 0, false, 0)
 	}
 	t.Log(manager.itemMap)
 }
@@ -19,7 +19,7 @@ func TestTrafficStatManager_Add(t *testing.T) {
 func TestTrafficStatManager_Upload(t *testing.T) {
 	manager := NewTrafficStatManager()
 	for i := 0; i < 100; i++ {
-		manager.Add(1, 1, "goedge.cn"+types.String(rands.Int(0, 10)), 1, 0, 1, 0, 0, 0, false, 0)
+		manager.Add(1, 1, "goedge.cn"+types.String(rands.Int(0, 10)), 1, 0, 1, 0, 0, 0, 0, false, 0)
 	}
 	err := manager.Upload()
 	if err != nil {
@@ -36,7 +36,7 @@ func BenchmarkTrafficStatManager_Add(b *testing.B) {
 
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			manager.Add(1, 1, "goedge.cn"+types.String(rand.Int63()%10), 1024, 1, 0, 0, 0, 0, false, 0)
+			manager.Add(1, 1, "goedge.cn"+types.String(rand.Int63()%10), 1024, 1, 0, 0, 0, 0, 0, false, 0)
 		}
 	})
 }

internal/stats/user_agent_parser.go

@@ -3,38 +3,51 @@
 package stats
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
+	syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync"
 	"github.com/mssola/useragent"
 	"sync"
+	"time"
 )
 
 var SharedUserAgentParser = NewUserAgentParser()
 
+const userAgentShardingCount = 8
+
 // UserAgentParser UserAgent解析器
 type UserAgentParser struct {
-	parser *useragent.UserAgent
+	cacheMaps [userAgentShardingCount]map[uint64]UserAgentParserResult
+	pool      *sync.Pool
+	mu        *syncutils.RWMutex
 
-	cacheMap1     map[uint64]UserAgentParserResult
-	cacheMap2     map[uint64]UserAgentParserResult
 	maxCacheItems int
-
-	cacheCursor int
-	locker      sync.RWMutex
+	gcTicker      *time.Ticker
+	gcIndex       int
 }
 
+// NewUserAgentParser 获取新解析器
 func NewUserAgentParser() *UserAgentParser {
 	var parser = &UserAgentParser{
-		parser:      &useragent.UserAgent{},
-		cacheMap1:   map[uint64]UserAgentParserResult{},
-		cacheMap2:   map[uint64]UserAgentParserResult{},
-		cacheCursor: 0,
+		pool: &sync.Pool{
+			New: func() any {
+				return &useragent.UserAgent{}
+			},
+		},
+		cacheMaps: [userAgentShardingCount]map[uint64]UserAgentParserResult{},
+		mu:        syncutils.NewRWMutex(userAgentShardingCount),
+	}
+
+	for i := 0; i < userAgentShardingCount; i++ {
+		parser.cacheMaps[i] = map[uint64]UserAgentParserResult{}
 	}
 
 	parser.init()
 	return parser
 }
 
+// 初始化
 func (this *UserAgentParser) init() {
 	var maxCacheItems = 10_000
 	var systemMemory = utils.SystemMemoryGB()
@@ -46,8 +59,16 @@ func (this *UserAgentParser) init() {
 		maxCacheItems = 20_000
 	}
 	this.maxCacheItems = maxCacheItems
+
+	this.gcTicker = time.NewTicker(5 * time.Second)
+	goman.New(func() {
+		for range this.gcTicker.C {
+			this.GC()
+		}
+	})
 }
 
+// Parse 解析UserAgent
 func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResult) {
 	// 限制长度
 	if len(userAgent) == 0 || len(userAgent) > 256 {
@@ -55,28 +76,22 @@ func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResu
 	}
 
 	var userAgentKey = fnv.HashString(userAgent)
+	var shardingIndex = int(userAgentKey % userAgentShardingCount)
 
-	this.locker.RLock()
-	cacheResult, ok := this.cacheMap1[userAgentKey]
+	this.mu.RLock(shardingIndex)
+	cacheResult, ok := this.cacheMaps[shardingIndex][userAgentKey]
 	if ok {
-		this.locker.RUnlock()
+		this.mu.RUnlock(shardingIndex)
 		return cacheResult
 	}
+	this.mu.RUnlock(shardingIndex)
 
-	cacheResult, ok = this.cacheMap2[userAgentKey]
-	if ok {
-		this.locker.RUnlock()
-		return cacheResult
-	}
-	this.locker.RUnlock()
-
-	this.locker.Lock()
-	defer this.locker.Unlock()
-
-	this.parser.Parse(userAgent)
-	result.OS = this.parser.OSInfo()
-	result.BrowserName, result.BrowserVersion = this.parser.Browser()
-	result.IsMobile = this.parser.Mobile()
+	var parser = this.pool.Get().(*useragent.UserAgent)
+	parser.Parse(userAgent)
+	result.OS = parser.OSInfo()
+	result.BrowserName, result.BrowserVersion = parser.Browser()
+	result.IsMobile = parser.Mobile()
+	this.pool.Put(parser)
 
 	// 忽略特殊字符
 	if len(result.BrowserName) > 0 {
@@ -87,19 +102,45 @@ func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResu
 		}
 	}
 
-	if this.cacheCursor == 0 {
-		this.cacheMap1[userAgentKey] = result
-		if len(this.cacheMap1) >= this.maxCacheItems {
-			this.cacheCursor = 1
-			this.cacheMap2 = map[uint64]UserAgentParserResult{}
-		}
-	} else {
-		this.cacheMap2[userAgentKey] = result
-		if len(this.cacheMap2) >= this.maxCacheItems {
-			this.cacheCursor = 0
-			this.cacheMap1 = map[uint64]UserAgentParserResult{}
-		}
-	}
+	this.mu.Lock(shardingIndex)
+	this.cacheMaps[shardingIndex][userAgentKey] = result
+	this.mu.Unlock(shardingIndex)
 
 	return
 }
+
+// MaxCacheItems 读取能容纳的缓存最大数量
+func (this *UserAgentParser) MaxCacheItems() int {
+	return this.maxCacheItems
+}
+
+// Len 读取当前缓存数量
+func (this *UserAgentParser) Len() int {
+	var total = 0
+	for i := 0; i < userAgentShardingCount; i++ {
+		this.mu.RLock(i)
+		total += len(this.cacheMaps[i])
+		this.mu.RUnlock(i)
+	}
+	return total
+}
+
+// GC 回收多余的缓存
+func (this *UserAgentParser) GC() {
+	var total = this.Len()
+	if total > this.maxCacheItems {
+		for {
+			var shardingIndex = this.gcIndex
+
+			this.mu.Lock(shardingIndex)
+			total -= len(this.cacheMaps[shardingIndex])
+			this.cacheMaps[shardingIndex] = map[uint64]UserAgentParserResult{}
+			this.gcIndex = (this.gcIndex + 1) % userAgentShardingCount
+			this.mu.Unlock(shardingIndex)
+
+			if total <= this.maxCacheItems {
+				break
+			}
+		}
+	}
+}

internal/stats/user_agent_parser_test.go

@@ -1,17 +1,21 @@
 // Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package stats
+package stats_test
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/stats"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
+	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
 	"runtime"
 	"runtime/debug"
 	"testing"
+	"time"
 )
 
 func TestUserAgentParser_Parse(t *testing.T) {
-	var parser = NewUserAgentParser()
+	var parser = stats.NewUserAgentParser()
 	for i := 0; i < 4; i++ {
 		t.Log(parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/1"))
 		t.Log(parser.Parse("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"))
@@ -19,7 +23,7 @@ func TestUserAgentParser_Parse(t *testing.T) {
 }
 
 func TestUserAgentParser_Parse_Unknown(t *testing.T) {
-	var parser = NewUserAgentParser()
+	var parser = stats.NewUserAgentParser()
 	t.Log(parser.Parse("Mozilla/5.0 (Wind 10.0; WOW64; rv:49.0) Apple/537.36 (KHTML, like Gecko) Chr/88.0.4324.96 Sa/537.36 Test/1"))
 	t.Log(parser.Parse(""))
 }
@@ -28,10 +32,10 @@ func TestUserAgentParser_Memory(t *testing.T) {
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
 
-	var parser = NewUserAgentParser()
+	var parser = stats.NewUserAgentParser()
 
 	for i := 0; i < 1_000_000; i++ {
-		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
+		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
 	}
 
 	runtime.GC()
@@ -40,32 +44,76 @@ func TestUserAgentParser_Memory(t *testing.T) {
 	var stat2 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat2)
 
-	t.Log("max cache items:", parser.maxCacheItems)
-	t.Log("cache1:", len(parser.cacheMap1), "cache2:", len(parser.cacheMap2), "cache3:", (stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
+	t.Log("max cache items:", parser.MaxCacheItems())
+	t.Log("cache:", parser.Len(), "usage:", (stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
 }
 
-func BenchmarkUserAgentParser_Parse(b *testing.B) {
-	var parser = NewUserAgentParser()
-	for i := 0; i < b.N; i++ {
-		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
+func TestNewUserAgentParser_GC(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
 	}
-	b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
-}
 
-func BenchmarkUserAgentParser_Parse2(b *testing.B) {
-	var parser = NewUserAgentParser()
-	for i := 0; i < b.N; i++ {
+	var parser = stats.NewUserAgentParser()
+
+	for i := 0; i < 1_000_000; i++ {
 		parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
 	}
-	b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
+
+	time.Sleep(60 * time.Second) // wait to gc
+	t.Log(parser.Len(), "cache items")
 }
 
-func BenchmarkUserAgentParser_Parse3(b *testing.B) {
-	var parser = NewUserAgentParser()
+func TestNewUserAgentParser_Mobile(t *testing.T) {
+	var a = assert.NewAssertion(t)
+	var parser = stats.NewUserAgentParser()
+	for _, userAgent := range []string{
+		"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",
+		"Mozilla/5.0 (Linux; U; Android 2.2.1; en-us; Nexus One Build/FRG83) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
+	} {
+		a.IsTrue(parser.Parse(userAgent).IsMobile)
+	}
+}
+
+func BenchmarkUserAgentParser_Parse_Many_LimitCPU(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var parser = stats.NewUserAgentParser()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
+		}
+	})
+	b.Log(parser.Len())
+}
+
+func BenchmarkUserAgentParser_Parse_Many(b *testing.B) {
+	var parser = stats.NewUserAgentParser()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 1_000_000)))
+		}
+	})
+	b.Log(parser.Len())
+}
+
+func BenchmarkUserAgentParser_Parse_Few_LimitCPU(b *testing.B) {
+	runtime.GOMAXPROCS(4)
+
+	var parser = stats.NewUserAgentParser()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
 		}
 	})
-	b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
+	b.Log(parser.Len())
+}
+
+func BenchmarkUserAgentParser_Parse_Few(b *testing.B) {
+	var parser = stats.NewUserAgentParser()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
+		}
+	})
+	b.Log(parser.Len())
 }

internal/ttlcache/cache_test.go

@@ -1,6 +1,7 @@
 package ttlcache
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
@@ -8,6 +9,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"runtime"
+	"runtime/debug"
 	"strconv"
 	"sync/atomic"
 	"testing"
@@ -48,16 +50,34 @@ func TestCache_Memory(t *testing.T) {
 	}
 
 	var cache = NewCache[int]()
+	var isReady bool
 
 	testutils.StartMemoryStats(t, func() {
+		if !isReady {
+			return
+		}
 		t.Log(cache.Count(), "items")
 	})
 
-	var count = 20_000_000
+	var count = 1_000_000
+	if utils.SystemMemoryGB() > 4 {
+		count = 20_000_000
+	}
 	for i := 0; i < count; i++ {
 		cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+int64(rands.Int(0, 300)))
 	}
 
+	func() {
+		var before = time.Now()
+		runtime.GC()
+		var costSeconds = time.Since(before).Seconds()
+		var stats = &debug.GCStats{}
+		debug.ReadGCStats(stats)
+		t.Log("GC pause:", stats.Pause[0].Seconds()*1000, "ms", "cost:", costSeconds*1000, "ms")
+	}()
+
+	isReady = true
+
 	t.Log(cache.Count())
 
 	time.Sleep(10 * time.Second)
@@ -105,6 +125,10 @@ func TestCache_IncreaseInt64(t *testing.T) {
 }
 
 func TestCache_Read(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	runtime.GOMAXPROCS(1)
 
 	var cache = NewCache[int](PiecesOption{Count: 32})

internal/utils/agents/manager_test.go

@@ -4,12 +4,17 @@ package agents_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/agents"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
 )
 
 func TestNewManager(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var db = agents.NewDB(Tea.Root + "/data/agents.db")
 	err := db.Init()
 	if err != nil {

internal/utils/agents/queue_test.go

@@ -4,6 +4,7 @@ package agents_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/agents"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"testing"
@@ -11,6 +12,10 @@ import (
 )
 
 func TestParseQueue_Process(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var queue = agents.NewQueue()
 	go queue.Start()
 	time.Sleep(1 * time.Second)
@@ -19,6 +24,10 @@ func TestParseQueue_Process(t *testing.T) {
 }
 
 func TestParseQueue_ParseIP(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	var queue = agents.NewQueue()
 	for _, ip := range []string{
 		"192.168.1.100",

internal/utils/cachehits/stat.go

@@ -6,6 +6,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"github.com/iwind/TeaGo/Tea"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -138,7 +139,7 @@ func (this *Stat) IsGood(category string) bool {
 			return true
 		}
 
-		if item.countCached > countSamples && item.timestamp < fasttime.Now().Unix()-600 /** 10 minutes ago **/ {
+		if item.countCached > countSamples && (Tea.IsTesting() || item.timestamp < fasttime.Now().Unix()-600) /** 10 minutes ago **/ {
 			var isGood = item.countHits*100/item.countCached >= this.goodRatio
 			if isGood {
 				item.isGood = true

internal/utils/cachehits/stat_test.go

@@ -58,10 +58,10 @@ func TestNewStat(t *testing.T) {
 
 	{
 		var stat = cachehits.NewStat(5)
-		for i := 0; i < 10001; i++ {
+		for i := 0; i < 100001; i++ {
 			stat.IncreaseCached("a")
 		}
-		for i := 0; i < 499; i++ {
+		for i := 0; i < 4999; i++ {
 			stat.IncreaseHit("a")
 		}
 

internal/utils/clock/manager_test.go

@@ -4,9 +4,14 @@ package clock_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"testing"
 )
 
 func TestReadServer(t *testing.T) {
+	if !testutils.IsSingleTesting() {
+		return
+	}
+
 	t.Log(clock.NewClockManager().ReadServer("pool.ntp.org"))
 }

internal/utils/counters/counter.go

@@ -13,12 +13,16 @@ import (
 
 const maxItemsPerGroup = 50_000
 
-var SharedCounter = NewCounter().WithGC()
+var SharedCounter = NewCounter[uint32]().WithGC()
 
-type Counter struct {
+type SupportedUIntType interface {
+	uint32 | uint64
+}
+
+type Counter[T SupportedUIntType] struct {
 	countMaps uint64
 	locker    *syncutils.RWMutex
-	itemMaps  []map[uint64]*Item
+	itemMaps  []map[uint64]Item[T]
 
 	gcTicker *time.Ticker
 	gcIndex  int
@@ -26,18 +30,18 @@ type Counter struct {
 }
 
 // NewCounter create new counter
-func NewCounter() *Counter {
+func NewCounter[T SupportedUIntType]() *Counter[T] {
 	var count = utils.SystemMemoryGB() * 8
 	if count < 8 {
 		count = 8
 	}
 
-	var itemMaps = []map[uint64]*Item{}
+	var itemMaps = []map[uint64]Item[T]{}
 	for i := 0; i < count; i++ {
-		itemMaps = append(itemMaps, map[uint64]*Item{})
+		itemMaps = append(itemMaps, map[uint64]Item[T]{})
 	}
 
-	var counter = &Counter{
+	var counter = &Counter[T]{
 		countMaps: uint64(count),
 		locker:    syncutils.NewRWMutex(count),
 		itemMaps:  itemMaps,
@@ -47,7 +51,7 @@ func NewCounter() *Counter {
 }
 
 // WithGC start the counter with gc automatically
-func (this *Counter) WithGC() *Counter {
+func (this *Counter[T]) WithGC() *Counter[T] {
 	if this.gcTicker != nil {
 		return this
 	}
@@ -62,62 +66,59 @@ func (this *Counter) WithGC() *Counter {
 }
 
 // Increase key
-func (this *Counter) Increase(key uint64, lifeSeconds int) uint64 {
+func (this *Counter[T]) Increase(key uint64, lifeSeconds int) T {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
-	var item = this.itemMaps[index][key]
+	var item = this.itemMaps[index][key] // item MUST NOT be pointer
 	this.locker.RUnlock(index)
-	if item == nil { // no need to care about duplication
-		item = NewItem(lifeSeconds)
+	if !item.IsOk() {
+		// no need to care about duplication
+		// always insert new item even when itemMap is full
+		item = NewItem[T](lifeSeconds)
+		var result = item.Increase()
 		this.locker.Lock(index)
-
-		// check again
-		oldItem, ok := this.itemMaps[index][key]
-		if !ok {
-			this.itemMaps[index][key] = item
-		} else {
-			item = oldItem
-		}
-
+		this.itemMaps[index][key] = item
 		this.locker.Unlock(index)
+		return result
 	}
 
 	this.locker.Lock(index)
 	var result = item.Increase()
+	this.itemMaps[index][key] = item // overwrite
 	this.locker.Unlock(index)
 	return result
 }
 
 // IncreaseKey increase string key
-func (this *Counter) IncreaseKey(key string, lifeSeconds int) uint64 {
+func (this *Counter[T]) IncreaseKey(key string, lifeSeconds int) T {
 	return this.Increase(this.hash(key), lifeSeconds)
 }
 
 // Get value of key
-func (this *Counter) Get(key uint64) uint64 {
+func (this *Counter[T]) Get(key uint64) T {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
 	defer this.locker.RUnlock(index)
 	var item = this.itemMaps[index][key]
-	if item != nil {
+	if item.IsOk() {
 		return item.Sum()
 	}
 	return 0
 }
 
 // GetKey get value of string key
-func (this *Counter) GetKey(key string) uint64 {
+func (this *Counter[T]) GetKey(key string) T {
 	return this.Get(this.hash(key))
 }
 
 // Reset key
-func (this *Counter) Reset(key uint64) {
+func (this *Counter[T]) Reset(key uint64) {
 	var index = int(key % this.countMaps)
 	this.locker.RLock(index)
 	var item = this.itemMaps[index][key]
 	this.locker.RUnlock(index)
 
-	if item != nil {
+	if item.IsOk() {
 		this.locker.Lock(index)
 		delete(this.itemMaps[index], key)
 		this.locker.Unlock(index)
@@ -125,12 +126,12 @@ func (this *Counter) Reset(key uint64) {
 }
 
 // ResetKey string key
-func (this *Counter) ResetKey(key string) {
+func (this *Counter[T]) ResetKey(key string) {
 	this.Reset(this.hash(key))
 }
 
 // TotalItems get items count
-func (this *Counter) TotalItems() int {
+func (this *Counter[T]) TotalItems() int {
 	var total = 0
 
 	for i := 0; i < int(this.countMaps); i++ {
@@ -143,7 +144,7 @@ func (this *Counter) TotalItems() int {
 }
 
 // GC garbage expired items
-func (this *Counter) GC() {
+func (this *Counter[T]) GC() {
 	this.gcLocker.Lock()
 	var gcIndex = this.gcIndex
 
@@ -192,11 +193,11 @@ func (this *Counter) GC() {
 	}
 }
 
-func (this *Counter) CountMaps() int {
+func (this *Counter[T]) CountMaps() int {
 	return int(this.countMaps)
 }
 
 // calculate hash of the key
-func (this *Counter) hash(key string) uint64 {
+func (this *Counter[T]) hash(key string) uint64 {
 	return xxhash.Sum64String(key)
 }

internal/utils/counters/counter_test.go

@@ -19,7 +19,7 @@ import (
 func TestCounter_Increase(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	a.IsTrue(counter.Increase(1, 10) == 1)
 	a.IsTrue(counter.Increase(1, 10) == 2)
 	a.IsTrue(counter.Increase(2, 10) == 1)
@@ -32,7 +32,7 @@ func TestCounter_Increase(t *testing.T) {
 func TestCounter_IncreaseKey(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	a.IsTrue(counter.IncreaseKey("1", 10) == 1)
 	a.IsTrue(counter.IncreaseKey("1", 10) == 2)
 	a.IsTrue(counter.IncreaseKey("2", 10) == 1)
@@ -47,13 +47,14 @@ func TestCounter_GC(t *testing.T) {
 		return
 	}
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 	counter.Increase(1, 20)
 	time.Sleep(1 * time.Second)
 	counter.Increase(1, 20)
 	time.Sleep(1 * time.Second)
 	counter.Increase(1, 20)
 	counter.GC()
+	t.Log(counter.Get(1))
 }
 
 func TestCounter_GC2(t *testing.T) {
@@ -61,8 +62,8 @@ func TestCounter_GC2(t *testing.T) {
 		return
 	}
 
-	var counter = counters.NewCounter().WithGC()
-	for i := 0; i < 1e5; i++ {
+	var counter = counters.NewCounter[uint32]().WithGC()
+	for i := 0; i < 100_000; i++ {
 		counter.Increase(uint64(i), rands.Int(10, 300))
 	}
 
@@ -79,7 +80,7 @@ func TestCounterMemory(t *testing.T) {
 	var stat = &runtime.MemStats{}
 	runtime.ReadMemStats(stat)
 
-	var counter = counters.NewCounter().WithGC()
+	var counter = counters.NewCounter[uint32]()
 	for i := 0; i < 1_000_000; i++ {
 		counter.Increase(uint64(i), rands.Int(10, 300))
 	}
@@ -90,21 +91,36 @@ func TestCounterMemory(t *testing.T) {
 
 	var stat1 = &runtime.MemStats{}
 	runtime.ReadMemStats(stat1)
-	t.Log((stat1.TotalAlloc-stat.TotalAlloc)/(1<<20), "MB")
+	t.Log((stat1.HeapInuse-stat.HeapInuse)/(1<<20), "MB")
 
 	t.Log(counter.TotalItems())
+
+	var gcPause = func() {
+		var before = time.Now()
+		runtime.GC()
+		var costSeconds = time.Since(before).Seconds()
+		var stats = &debug.GCStats{}
+		debug.ReadGCStats(stats)
+		t.Log("GC pause:", stats.Pause[0].Seconds()*1000, "ms", "cost:", costSeconds*1000, "ms")
+	}
+
+	gcPause()
+
+	_ = counter.TotalItems()
 }
 
 func BenchmarkCounter_Increase(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
+
+	b.ReportAllocs()
 	b.ResetTimer()
 
 	var i uint64
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			counter.Increase(atomic.AddUint64(&i, 1)%1e6, 20)
+			counter.Increase(atomic.AddUint64(&i, 1)%1_000_000, 20)
 		}
 	})
 
@@ -114,7 +130,7 @@ func BenchmarkCounter_Increase(b *testing.B) {
 func BenchmarkCounter_IncreaseKey(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 
 	go func() {
 		var ticker = time.NewTicker(100 * time.Millisecond)
@@ -124,11 +140,12 @@ func BenchmarkCounter_IncreaseKey(b *testing.B) {
 	}()
 
 	b.ResetTimer()
+	b.ReportAllocs()
 
 	var i uint64
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			counter.IncreaseKey(types.String(atomic.AddUint64(&i, 1)%1e6), 20)
+			counter.IncreaseKey(types.String(atomic.AddUint64(&i, 1)%1_000_000), 20)
 		}
 	})
 
@@ -138,7 +155,7 @@ func BenchmarkCounter_IncreaseKey(b *testing.B) {
 func BenchmarkCounter_IncreaseKey2(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 
 	go func() {
 		var ticker = time.NewTicker(1 * time.Millisecond)
@@ -162,7 +179,7 @@ func BenchmarkCounter_IncreaseKey2(b *testing.B) {
 func BenchmarkCounter_GC(b *testing.B) {
 	runtime.GOMAXPROCS(4)
 
-	var counter = counters.NewCounter()
+	var counter = counters.NewCounter[uint32]()
 
 	for i := uint64(0); i < 1e5; i++ {
 		counter.IncreaseKey(types.String(i), 20)

internal/utils/counters/item.go

@@ -6,40 +6,43 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
 )
 
-type Item struct {
-	lifeSeconds int64
-
-	spanSeconds int64
-	spans       []uint64
+const spanMaxValue = 10_000_000
+const maxSpans = 10
 
+type Item[T SupportedUIntType] struct {
+	spans          [maxSpans + 1]T
 	lastUpdateTime int64
+	lifeSeconds    int64
+	spanSeconds    int64
 }
 
-func NewItem(lifeSeconds int) *Item {
+func NewItem[T SupportedUIntType](lifeSeconds int) Item[T] {
 	if lifeSeconds <= 0 {
 		lifeSeconds = 60
 	}
-	var spanSeconds = lifeSeconds / 10
+	var spanSeconds = lifeSeconds / maxSpans
 	if spanSeconds < 1 {
 		spanSeconds = 1
+	} else if lifeSeconds > maxSpans && lifeSeconds%maxSpans != 0 {
+		spanSeconds++
 	}
-	var countSpans = lifeSeconds/spanSeconds + 1 /** prevent index out of bounds **/
 
-	return &Item{
+	return Item[T]{
 		lifeSeconds:    int64(lifeSeconds),
 		spanSeconds:    int64(spanSeconds),
-		spans:          make([]uint64, countSpans),
 		lastUpdateTime: fasttime.Now().Unix(),
 	}
 }
 
-func (this *Item) Increase() (result uint64) {
+func (this *Item[T]) Increase() (result T) {
 	var currentTime = fasttime.Now().Unix()
 	var currentSpanIndex = this.calculateSpanIndex(currentTime)
 
 	// return quickly
 	if this.lastUpdateTime == currentTime {
-		this.spans[currentSpanIndex]++
+		if this.spans[currentSpanIndex] < spanMaxValue {
+			this.spans[currentSpanIndex]++
+		}
 		for _, count := range this.spans {
 			result += count
 		}
@@ -69,7 +72,9 @@ func (this *Item) Increase() (result uint64) {
 		}
 	}
 
-	this.spans[currentSpanIndex]++
+	if this.spans[currentSpanIndex] < spanMaxValue {
+		this.spans[currentSpanIndex]++
+	}
 	this.lastUpdateTime = currentTime
 
 	for _, count := range this.spans {
@@ -79,7 +84,7 @@ func (this *Item) Increase() (result uint64) {
 	return
 }
 
-func (this *Item) Sum() (result uint64) {
+func (this *Item[T]) Sum() (result T) {
 	if this.lastUpdateTime == 0 {
 		return 0
 	}
@@ -104,16 +109,24 @@ func (this *Item) Sum() (result uint64) {
 	return result
 }
 
-func (this *Item) Reset() {
+func (this *Item[T]) Reset() {
 	for index := range this.spans {
 		this.spans[index] = 0
 	}
 }
 
-func (this *Item) IsExpired(currentTime int64) bool {
+func (this *Item[T]) IsExpired(currentTime int64) bool {
 	return this.lastUpdateTime < currentTime-this.lifeSeconds-this.spanSeconds
 }
 
-func (this *Item) calculateSpanIndex(timestamp int64) int {
-	return int(timestamp % this.lifeSeconds / this.spanSeconds)
+func (this *Item[T]) calculateSpanIndex(timestamp int64) int {
+	var index = int(timestamp % this.lifeSeconds / this.spanSeconds)
+	if index > maxSpans-1 {
+		return maxSpans - 1
+	}
+	return index
+}
+
+func (this *Item[T]) IsOk() bool {
+	return this.lifeSeconds > 0
 }

internal/utils/counters/item_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/counters"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
 	"github.com/iwind/TeaGo/assert"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"runtime"
 	"testing"
@@ -17,7 +18,7 @@ func TestItem_Increase(t *testing.T) {
 		return
 	}
 
-	var item = counters.NewItem(10)
+	var item = counters.NewItem[uint32](10)
 	t.Log(item.Increase(), item.Sum())
 	time.Sleep(1 * time.Second)
 	t.Log(item.Increase(), item.Sum())
@@ -41,9 +42,9 @@ func TestItem_Increase2(t *testing.T) {
 
 	var a = assert.NewAssertion(t)
 
-	var item = counters.NewItem(20)
+	var item = counters.NewItem[uint32](23)
 	for i := 0; i < 100; i++ {
-		t.Log(item.Increase(), item.Sum(), timeutil.Format("H:i:s"))
+		t.Log("round "+types.String(i)+":", item.Increase(), item.Sum(), timeutil.Format("H:i:s"))
 		time.Sleep(2 * time.Second)
 	}
 
@@ -56,14 +57,14 @@ func TestItem_IsExpired(t *testing.T) {
 		return
 	}
 
-	var currentTime = time.Now().Unix()
-
-	var item = counters.NewItem(10)
-	t.Log(item.IsExpired(currentTime))
+	var item = counters.NewItem[uint32](10)
+	t.Log(item.IsExpired(time.Now().Unix()))
 	time.Sleep(10 * time.Second)
-	t.Log(item.IsExpired(currentTime))
+	t.Log(item.IsExpired(time.Now().Unix()))
 	time.Sleep(2 * time.Second)
-	t.Log(item.IsExpired(currentTime))
+	t.Log(item.IsExpired(time.Now().Unix()))
+	time.Sleep(2 * time.Second)
+	t.Log(item.IsExpired(time.Now().Unix()))
 }
 
 func BenchmarkItem_Increase(b *testing.B) {
@@ -73,7 +74,7 @@ func BenchmarkItem_Increase(b *testing.B) {
 
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			var item = counters.NewItem(60)
+			var item = counters.NewItem[uint32](60)
 			item.Increase()
 			item.Sum()
 		}

internal/utils/ip_test.go

@@ -42,12 +42,12 @@ func TestIsIPv4(t *testing.T) {
 func TestIsIPv6(t *testing.T) {
 	var a = assert.NewAssertion(t)
 	a.IsFalse(utils.IsIPv6("192.168.1.1"))
-	a.IsFloat32(utils.IsIPv6("0.0.0.0"))
+	a.IsFalse(utils.IsIPv6("0.0.0.0"))
 	a.IsFalse(utils.IsIPv6("192.168.1.256"))
 	a.IsFalse(utils.IsIPv6("192.168.1"))
 	a.IsTrue(utils.IsIPv6("::1"))
 	a.IsTrue(utils.IsIPv6("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
-	a.IsTrue(utils.IsIPv4("::ffff:192.168.0.1"))
+	a.IsFalse(utils.IsIPv4("::ffff:192.168.0.1"))
 	a.IsTrue(utils.IsIPv6("::ffff:192.168.0.1"))
 }
 

internal/utils/linkedlist/list.go

@@ -2,7 +2,7 @@
 
 package linkedlist
 
-type List[T any]  struct {
+type List[T any] struct {
 	head  *Item[T]
 	end   *Item[T]
 	count int
@@ -36,6 +36,15 @@ func (this *List[T]) Push(item *Item[T]) {
 	this.add(item)
 }
 
+func (this *List[T]) Shift() *Item[T] {
+	if this.head != nil {
+		var old = this.head
+		this.Remove(this.head)
+		return old
+	}
+	return nil
+}
+
 func (this *List[T]) Remove(item *Item[T]) {
 	if item == nil {
 		return
@@ -71,6 +80,15 @@ func (this *List[T]) Range(f func(item *Item[T]) (goNext bool)) {
 	}
 }
 
+func (this *List[T]) RangeReverse(f func(item *Item[T]) (goNext bool)) {
+	for e := this.end; e != nil; e = e.prev {
+		goNext := f(e)
+		if !goNext {
+			break
+		}
+	}
+}
+
 func (this *List[T]) Reset() {
 	this.head = nil
 	this.end = nil

internal/utils/linkedlist/list_test.go

@@ -4,6 +4,7 @@ package linkedlist_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
+	"github.com/iwind/TeaGo/types"
 	"runtime"
 	"strconv"
 	"testing"
@@ -95,6 +96,48 @@ func TestList_Push(t *testing.T) {
 	})
 }
 
+func TestList_Shift(t *testing.T) {
+	var list = linkedlist.NewList[int]()
+	list.Push(linkedlist.NewItem(1))
+	list.Push(linkedlist.NewItem(2))
+	list.Push(linkedlist.NewItem(3))
+	list.Push(linkedlist.NewItem(4))
+
+	for i := 0; i < 10; i++ {
+		t.Log("=== before shift " + types.String(i) + " ===")
+		list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
+			t.Log(item.Value)
+			return true
+		})
+
+		t.Logf("shift: %+v", list.Shift())
+
+		t.Log("=== after shift  " + types.String(i) + " ===")
+		list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
+			t.Log(item.Value)
+			return true
+		})
+	}
+}
+
+func TestList_RangeReverse(t *testing.T) {
+	var list = linkedlist.NewList[int]()
+	list.Push(linkedlist.NewItem(1))
+	list.Push(linkedlist.NewItem(2))
+
+	var item3 = linkedlist.NewItem(3)
+	list.Push(item3)
+
+	list.Push(linkedlist.NewItem(4))
+
+	//list.Push(item3)
+	//list.Remove(item3)
+	list.RangeReverse(func(item *linkedlist.Item[int]) (goNext bool) {
+		t.Log(item.Value)
+		return true
+	})
+}
+
 func BenchmarkList_Add(b *testing.B) {
 	var list = linkedlist.NewList[int]()
 	for i := 0; i < b.N; i++ {

internal/utils/ratelimit/bandwidth.go

@@ -0,0 +1,61 @@
+// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package ratelimit
+
+import (
+	"context"
+	"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
+	"sync/atomic"
+	"time"
+)
+
+// Bandwidth lossy bandwidth limiter
+type Bandwidth struct {
+	totalBytes int64
+
+	currentTimestamp int64
+	currentBytes     int64
+}
+
+// NewBandwidth create new bandwidth limiter
+func NewBandwidth(totalBytes int64) *Bandwidth {
+	return &Bandwidth{totalBytes: totalBytes}
+}
+
+// Ack acquire next chance to send data
+func (this *Bandwidth) Ack(ctx context.Context, newBytes int) {
+	if newBytes <= 0 {
+		return
+	}
+	if this.totalBytes <= 0 {
+		return
+	}
+
+	var timestamp = fasttime.Now().Unix()
+	if this.currentTimestamp != 0 && this.currentTimestamp != timestamp {
+		this.currentTimestamp = timestamp
+		this.currentBytes = int64(newBytes)
+
+		// 第一次发送直接放行，不需要判断
+		return
+	}
+
+	if this.currentTimestamp == 0 {
+		this.currentTimestamp = timestamp
+	}
+	if atomic.AddInt64(&this.currentBytes, int64(newBytes)) <= this.totalBytes {
+		return
+	}
+
+	var timeout = time.NewTimer(1 * time.Second)
+	if ctx != nil {
+		select {
+		case <-timeout.C:
+		case <-ctx.Done():
+		}
+	} else {
+		select {
+		case <-timeout.C:
+		}
+	}
+}