Compare commits
152 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17af07cce0 | ||
|
|
cfa57fac66 | ||
|
|
47523eaa73 | ||
|
|
27a24c6a8a | ||
|
|
9bc2b1a651 | ||
|
|
4f24b7f39c | ||
|
|
4607a1f4e7 | ||
|
|
0f2068b161 | ||
|
|
c039691a71 | ||
|
|
930ee44065 | ||
|
|
8a9aac7d72 | ||
|
|
e50bbb962d | ||
|
|
9ff936d0c1 | ||
|
|
f53727b09c | ||
|
|
525ce1f923 | ||
|
|
16e7cd800c | ||
|
|
3f34bfc0b0 | ||
|
|
548cd1002b | ||
|
|
3423865868 | ||
|
|
037bc8e0de | ||
|
|
e03292de28 | ||
|
|
ee2565905e | ||
|
|
05881b457d | ||
|
|
b116effc6c | ||
|
|
536efeeb9c | ||
|
|
e8638e4bec | ||
|
|
c9db722129 | ||
|
|
90de472bd5 | ||
|
|
50c6c60abf | ||
|
|
cc10372fe1 | ||
|
|
05c98a0656 | ||
|
|
1a790fe391 | ||
|
|
7dbd73cb59 | ||
|
|
4dfa571547 | ||
|
|
9f77f62308 | ||
|
|
facea1ed96 | ||
|
|
e367814db3 | ||
|
|
3a15408c98 | ||
|
|
c504b37118 | ||
|
|
74708dc02f | ||
|
|
0c097498bb | ||
|
|
981c063eff | ||
|
|
5e35c50113 | ||
|
|
e6c2869ff2 | ||
|
|
358bec2e9b | ||
|
|
1cd644f2eb | ||
|
|
f783e5c331 | ||
|
|
c39b1c794f | ||
|
|
2633d43897 | ||
|
|
88dca006c4 | ||
|
|
98feb26b79 | ||
|
|
ac6683e79d | ||
|
|
99d24afbcd | ||
|
|
ba19a9f4c4 | ||
|
|
7fea67a2b5 | ||
|
|
ecd2e6955e | ||
|
|
09d60a3047 | ||
|
|
e24f390412 | ||
|
|
eeacec1a4e | ||
|
|
30cd6373c5 | ||
|
|
87a6ab0559 | ||
|
|
59f27215d3 | ||
|
|
768384dcf0 | ||
|
|
3b52ac0fd2 | ||
|
|
41343b2264 | ||
|
|
d084059f04 | ||
|
|
9253c44ba5 | ||
|
|
ddec0bf2e0 | ||
|
|
aeba1805af | ||
|
|
ecff37e080 | ||
|
|
d31dac75be | ||
|
|
4571c84102 | ||
|
|
6a9f59bee0 | ||
|
|
f1951869f1 | ||
|
|
cfd4195c0f | ||
|
|
d793472b42 | ||
|
|
1e56247b9c | ||
|
|
c34a38857a | ||
|
|
57fa7036dc | ||
|
|
b8a3ac750f | ||
|
|
9d6692db0c | ||
|
|
ad94327226 | ||
|
|
aee1ff9609 | ||
|
|
822e967874 | ||
|
|
2acf890b8e | ||
|
|
3909695b44 | ||
|
|
d0f420a945 | ||
|
|
7618338f38 | ||
|
|
9b2a704e7f | ||
|
|
630c1ec63b | ||
|
|
5b93b28690 | ||
|
|
3aa68b5ffc | ||
|
|
adb0069c59 | ||
|
|
211da66226 | ||
|
|
81911c4073 | ||
|
|
6ff3230bab | ||
|
|
f01eae3590 | ||
|
|
47e8761209 | ||
|
|
8449fe7c0b | ||
|
|
7f3e6ddc65 | ||
|
|
6ca8b6837c | ||
|
|
a9969430a3 | ||
|
|
7c5c06191d | ||
|
|
afc533c3e4 | ||
|
|
71c891ae14 | ||
|
|
e8570bfd09 | ||
|
|
534d64673f | ||
|
|
6bff5c978b | ||
|
|
38a214878a | ||
|
|
149e04d0e4 | ||
|
|
5733d466ca | ||
|
|
a95e2e3259 | ||
|
|
a80a89edf5 | ||
|
|
b8f7d4110f | ||
|
|
00e76a6a09 | ||
|
|
79fa9d88a1 | ||
|
|
c460421279 | ||
|
|
69e4dd6cfe | ||
|
|
b73f0ae2c9 | ||
|
|
2af577380e | ||
|
|
89bfdc478f | ||
|
|
b6e8221ac1 | ||
|
|
ea6d3d7107 | ||
|
|
7d8bdfcd45 | ||
|
|
70fe1b5d2b | ||
|
|
a7caf0260a | ||
|
|
d547793eee | ||
|
|
d92f27c44b | ||
|
|
8561ff3e2d | ||
|
|
0736b20d33 | ||
|
|
263acb775b | ||
|
|
7a1fff8504 | ||
|
|
5c5adf690f | ||
|
|
1eca5099df | ||
|
|
5f2ad8b096 | ||
|
|
4405cfd405 | ||
|
|
2f6aa0c14f | ||
|
|
bb50ecd682 | ||
|
|
ae1454f9bb | ||
|
|
3781b1920a | ||
|
|
125ad9c606 | ||
|
|
e516300dc7 | ||
|
|
6c1d24c3e5 | ||
|
|
6f230b30e0 | ||
|
|
8a0318b4f3 | ||
|
|
d9fddcb001 | ||
|
|
9113c4c1b3 | ||
|
|
f0762fe1b9 | ||
|
|
9054b8ec05 | ||
|
|
5ad25e34c6 | ||
|
|
d4cca10301 | ||
|
|
8597884ec5 |
@@ -6,10 +6,11 @@ function build() {
|
||||
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
|
||||
DIST=$ROOT/"../dist/${NAME}"
|
||||
MUSL_DIR="/usr/local/opt/musl-cross/bin"
|
||||
SRCDIR=$(realpath "$ROOT/..")
|
||||
|
||||
# for macOS users: precompiled gcc can be downloaded from https://github.com/messense/homebrew-macos-cross-toolchains
|
||||
GCC_X86_64_DIR="/usr/local/gcc/x86_64-unknown-linux-gnu/bin"
|
||||
GCC_ARM64_DIR="//usr/local/gcc/aarch64-unknown-linux-gnu/bin"
|
||||
GCC_ARM64_DIR="/usr/local/gcc/aarch64-unknown-linux-gnu/bin"
|
||||
|
||||
OS=${1}
|
||||
ARCH=${2}
|
||||
@@ -70,6 +71,8 @@ function build() {
|
||||
|
||||
CC_PATH=""
|
||||
CXX_PATH=""
|
||||
CGO_LDFLAGS=""
|
||||
CGO_CFLAGS=""
|
||||
BUILD_TAG=$TAG
|
||||
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
|
||||
if [ "${ARCH}" == "amd64" ]; then
|
||||
@@ -79,7 +82,7 @@ function build() {
|
||||
CC_PATH="x86_64-unknown-linux-gnu-gcc"
|
||||
CXX_PATH="x86_64-unknown-linux-gnu-g++"
|
||||
if [ "$TAG" = "plus" ]; then
|
||||
BUILD_TAG="plus,script"
|
||||
BUILD_TAG="plus,script,packet"
|
||||
fi
|
||||
else
|
||||
CC_PATH="x86_64-linux-musl-gcc"
|
||||
@@ -97,7 +100,7 @@ function build() {
|
||||
CC_PATH="aarch64-unknown-linux-gnu-gcc"
|
||||
CXX_PATH="aarch64-unknown-linux-gnu-g++"
|
||||
if [ "$TAG" = "plus" ]; then
|
||||
BUILD_TAG="plus,script"
|
||||
BUILD_TAG="plus,script,packet"
|
||||
fi
|
||||
else
|
||||
CC_PATH="aarch64-linux-musl-gcc"
|
||||
@@ -117,13 +120,26 @@ function build() {
|
||||
CXX_PATH="mips64el-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
|
||||
# libpcap
|
||||
if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
|
||||
CGO_LDFLAGS="-L${SRCDIR}/libs/libpcap/${ARCH} -lpcap -L${SRCDIR}/libs/libbrotli/${ARCH} -lbrotlienc -lbrotlidec -lbrotlicommon"
|
||||
CGO_CFLAGS="-I${SRCDIR}/libs/libpcap/src/libpcap -I${SRCDIR}/libs/libpcap/src/libpcap/pcap -I${SRCDIR}/libs/libbrotli/src/brotli/c/include"
|
||||
fi
|
||||
|
||||
if [ ! -z $CC_PATH ]; then
|
||||
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
env CC=$MUSL_DIR/$CC_PATH \
|
||||
CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" \
|
||||
GOARCH="${ARCH}" CGO_ENABLED=1 \
|
||||
CGO_LDFLAGS="${CGO_LDFLAGS}" \
|
||||
CGO_CFLAGS="${CGO_CFLAGS}" \
|
||||
go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
else
|
||||
if [[ `uname` == *"Linux"* ]] && [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
|
||||
BUILD_TAG="plus,script"
|
||||
BUILD_TAG="plus,script,packet"
|
||||
fi
|
||||
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
|
||||
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
fi
|
||||
|
||||
if [ ! -f "${DIST}/bin/${NAME}" ]; then
|
||||
|
||||
@@ -228,11 +228,18 @@ func main() {
|
||||
})
|
||||
app.On("gc", func() {
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
_, err := sock.Send(&gosock.Command{Code: "gc"})
|
||||
reply, err := sock.Send(&gosock.Command{Code: "gc"})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
} else {
|
||||
fmt.Println("ok")
|
||||
if reply == nil {
|
||||
fmt.Println("ok")
|
||||
} else {
|
||||
var paramMap = maps.NewMap(reply.Params)
|
||||
var pauseMS = paramMap.GetFloat64("pauseMS")
|
||||
var costMS = paramMap.GetFloat64("costMS")
|
||||
fmt.Printf("ok, cost: %.4fms, pause: %.4fms", costMS, pauseMS)
|
||||
}
|
||||
}
|
||||
})
|
||||
app.On("ip.drop", func() {
|
||||
@@ -475,6 +482,19 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
var progressSock = gosock.NewTmpSock(teaconst.CacheGarbageSockName)
|
||||
progressSock.OnCommand(func(cmd *gosock.Command) {
|
||||
var params = maps.NewMap(cmd.Params)
|
||||
if cmd.Code == "progress" {
|
||||
fmt.Printf("%.2f%% %d\n", params.GetFloat64("progress")*100, params.GetInt("count"))
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
})
|
||||
go func() {
|
||||
_ = progressSock.Listen()
|
||||
}()
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{
|
||||
Code: "cache.garbage",
|
||||
|
||||
40
go.mod
40
go.mod
@@ -19,42 +19,41 @@ require (
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
|
||||
github.com/fsnotify/fsnotify v1.6.0
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/google/nftables v0.1.0
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible
|
||||
github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
|
||||
github.com/iwind/gowebp v0.0.0-20230911074406-2e4e7fd0b59f
|
||||
github.com/klauspost/compress v1.16.5
|
||||
github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4
|
||||
github.com/klauspost/compress v1.17.2
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mdlayher/netlink v1.7.1
|
||||
github.com/miekg/dns v1.1.43
|
||||
github.com/mssola/useragent v1.0.0
|
||||
github.com/pires/go-proxyproto v0.6.1
|
||||
github.com/qiniu/go-sdk/v7 v7.16.0
|
||||
github.com/quic-go/quic-go v0.38.1
|
||||
github.com/quic-go/quic-go v0.39.2
|
||||
github.com/shirou/gopsutil/v3 v3.22.2
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.41
|
||||
golang.org/x/image v0.7.0
|
||||
golang.org/x/net v0.14.0
|
||||
golang.org/x/sys v0.11.0
|
||||
google.golang.org/grpc v1.55.0
|
||||
google.golang.org/protobuf v1.30.0
|
||||
golang.org/x/image v0.13.0
|
||||
golang.org/x/net v0.17.0
|
||||
golang.org/x/sys v0.13.0
|
||||
google.golang.org/grpc v1.59.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/chai2010/webp v1.1.1 // indirect
|
||||
github.com/clbanning/mxj v1.8.4 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/golang/mock v1.6.0 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/go-querystring v1.0.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/native v1.0.0 // indirect
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
|
||||
@@ -62,21 +61,22 @@ require (
|
||||
github.com/mdlayher/socket v0.4.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.12.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/quic-go/qpack v0.4.0 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
|
||||
github.com/tdewolff/minify/v2 v2.12.7 // indirect
|
||||
github.com/tdewolff/parse/v2 v2.6.6 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.9 // indirect
|
||||
github.com/tklauser/numcpus v0.3.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
golang.org/x/crypto v0.12.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
|
||||
golang.org/x/mod v0.12.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/text v0.12.0 // indirect
|
||||
go.uber.org/mock v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/sync v0.4.0 // indirect
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
golang.org/x/tools v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
|
||||
)
|
||||
|
||||
113
go.sum
113
go.sum
@@ -15,8 +15,6 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
|
||||
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
|
||||
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
|
||||
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
|
||||
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
|
||||
@@ -43,8 +41,6 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
@@ -55,12 +51,12 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo=
|
||||
github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ=
|
||||
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
|
||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0=
|
||||
github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible h1:XRAk4HBDLCYEdPLWtKf5iZhOi7lfx17aY0oSO9+mcg8=
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
|
||||
github.com/iwind/TeaGo v0.0.0-20230630104525-161f0b32996d h1:XnTIj781NdSipts60fVqbgZorVAKVSRaA6nqVNfBQ1g=
|
||||
@@ -71,10 +67,8 @@ github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedV
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
|
||||
github.com/iwind/gowebp v0.0.0-20230615040911-5013dbb9d508 h1:fjKiHAyPQmdwuw1DQ2BI1JTbhUWAtI3Kr9wIZQBdRgQ=
|
||||
github.com/iwind/gowebp v0.0.0-20230615040911-5013dbb9d508/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
|
||||
github.com/iwind/gowebp v0.0.0-20230911074406-2e4e7fd0b59f h1:b+YNSK4PgRU4u5YuYW8W4dHO3LNsG7XvX2dJQK0jOf8=
|
||||
github.com/iwind/gowebp v0.0.0-20230911074406-2e4e7fd0b59f/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
|
||||
github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4 h1:eyymORsZg0tZ0niyolYF4nao4sdNUI+Ll40s96tKHBY=
|
||||
github.com/iwind/gowebp v0.0.0-20231026013903-1c22b0d78cc4/go.mod h1:AYyXDhbbD7q9N6rJff2jrE7pGupaiyvtv3YeyIAQLXk=
|
||||
github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4 h1:RPAH9Sj9l/20zH5zU5/iJGszfwPq6eLjoiC/n/asulA=
|
||||
github.com/iwind/nftables v0.0.0-20230419014751-9f023a644ad4/go.mod h1:7OLL+86wZKfBnAJxNxmdcZ0ebbgdp/A28fcagx9oJqA=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
@@ -85,8 +79,8 @@ github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTx
|
||||
github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
|
||||
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
|
||||
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
@@ -115,11 +109,8 @@ github.com/mssola/useragent v1.0.0 h1:WRlDpXyxHDNfvZaPEut5Biveq86Ze4o4EMffyMxmH5
|
||||
github.com/mssola/useragent v1.0.0/go.mod h1:hz9Cqz4RXusgg1EdI4Al0INR62kP7aPSRNHnpU+b85Y=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
|
||||
github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
|
||||
github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI=
|
||||
github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
|
||||
github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
|
||||
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
|
||||
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
|
||||
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
|
||||
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
|
||||
@@ -135,14 +126,10 @@ github.com/qiniu/go-sdk/v7 v7.16.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYX
|
||||
github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs=
|
||||
github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
|
||||
github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
|
||||
github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg=
|
||||
github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE=
|
||||
github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
|
||||
github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/quic-go v0.39.2 h1:hmwAf8zAHlvan0Y5PXxeeBFZEW17IW99sXLry8I2kjk=
|
||||
github.com/quic-go/quic-go v0.39.2/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
@@ -172,49 +159,45 @@ github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ
|
||||
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
|
||||
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
|
||||
github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
|
||||
go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
|
||||
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
|
||||
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
|
||||
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
|
||||
golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
|
||||
golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
|
||||
golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
|
||||
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
|
||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
|
||||
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -223,45 +206,39 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
|
||||
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E=
|
||||
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
|
||||
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
|
||||
google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
|
||||
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
|
||||
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
|
||||
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
|
||||
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
@@ -274,5 +251,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rogchap.com/v8go v0.9.0 h1:wYbUCO4h6fjTamziHrzyrPnpFNuzPpjZY+nfmZjNaew=
|
||||
rogchap.com/v8go v0.9.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs=
|
||||
|
||||
@@ -9,9 +9,13 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -25,7 +29,7 @@ type FileList struct {
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
|
||||
memoryCache *ttlcache.Cache
|
||||
memoryCache *ttlcache.Cache[zero.Zero]
|
||||
|
||||
// 老数据库地址
|
||||
oldDir string
|
||||
@@ -34,7 +38,7 @@ type FileList struct {
|
||||
func NewFileList(dir string) ListInterface {
|
||||
return &FileList{
|
||||
dir: dir,
|
||||
memoryCache: ttlcache.NewCache(),
|
||||
memoryCache: ttlcache.NewCache[zero.Zero](),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,19 +64,37 @@ func (this *FileList) Init() error {
|
||||
}
|
||||
|
||||
remotelogs.Println("CACHE", "loading database from '"+dir+"' ...")
|
||||
var wg = &sync.WaitGroup{}
|
||||
var locker = sync.Mutex{}
|
||||
var lastErr error
|
||||
|
||||
for i := 0; i < CountFileDB; i++ {
|
||||
var db = NewFileListDB()
|
||||
err = db.Open(dir + "/db-" + types.String(i) + ".db")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
|
||||
err = db.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var db = NewFileListDB()
|
||||
dbErr := db.Open(dir + "/db-" + types.String(i) + ".db")
|
||||
if dbErr != nil {
|
||||
lastErr = dbErr
|
||||
return
|
||||
}
|
||||
|
||||
this.dbList[i] = db
|
||||
dbErr = db.Init()
|
||||
if dbErr != nil {
|
||||
lastErr = dbErr
|
||||
return
|
||||
}
|
||||
|
||||
locker.Lock()
|
||||
this.dbList[i] = db
|
||||
locker.Unlock()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if lastErr != nil {
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// 升级老版本数据库
|
||||
@@ -100,9 +122,7 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// 这里不增加点击量,以减少对数据库的操作次数
|
||||
|
||||
this.memoryCache.Write(hash, 1, item.ExpiredAt)
|
||||
this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(item.ExpiredAt))
|
||||
|
||||
if this.onAdd != nil {
|
||||
this.onAdd(item)
|
||||
@@ -140,7 +160,7 @@ func (this *FileList) Exist(hash string) (bool, error) {
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
this.memoryCache.Write(hash, 1, expiredAt)
|
||||
this.memoryCache.Write(hash, zero.Zero{}, this.maxExpiresAtForMemoryCache(expiredAt))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -217,7 +237,7 @@ func (this *FileList) CleanMatchPrefix(prefix string) error {
|
||||
}
|
||||
|
||||
func (this *FileList) Remove(hash string) error {
|
||||
_, err := this.remove(hash)
|
||||
_, err := this.remove(hash, false)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -236,11 +256,16 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
|
||||
if err != nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if len(hashStrings) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
countFound += len(hashStrings)
|
||||
|
||||
// 不在 rows.Next() 循环中操作是为了避免死锁
|
||||
for _, hash := range hashStrings {
|
||||
err = this.Remove(hash)
|
||||
_, err = this.remove(hash, true)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -250,6 +275,11 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = db.writeDB.Exec(`DELETE FROM "cacheItems" WHERE "hash" IN ('` + strings.Join(hashStrings, "', '") + `')`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
return countFound, nil
|
||||
@@ -267,9 +297,13 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
|
||||
return err
|
||||
}
|
||||
|
||||
if len(hashStrings) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 不在 rows.Next() 循环中操作是为了避免死锁
|
||||
for _, hash := range hashStrings {
|
||||
_, err = this.remove(hash)
|
||||
_, err = this.remove(hash, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -279,6 +313,11 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = db.writeDB.Exec(`DELETE FROM "cacheItems" WHERE "hash" IN ('` + strings.Join(hashStrings, "', '") + `')`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -388,7 +427,7 @@ func (this *FileList) HashMapIsLoaded() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *FileList) remove(hash string) (notFound bool, err error) {
|
||||
func (this *FileList) remove(hash string, isDeleted bool) (notFound bool, err error) {
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
@@ -404,9 +443,11 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
|
||||
// 从缓存中删除
|
||||
this.memoryCache.Delete(hash)
|
||||
|
||||
err = db.DeleteSync(hash)
|
||||
if err != nil {
|
||||
return false, db.WrapError(err)
|
||||
if !isDeleted {
|
||||
err = db.DeleteSync(hash)
|
||||
if err != nil {
|
||||
return false, db.WrapError(err)
|
||||
}
|
||||
}
|
||||
|
||||
if this.onRemove != nil {
|
||||
@@ -439,7 +480,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
remotelogs.Println("CACHE", "upgrading local database finished")
|
||||
}()
|
||||
|
||||
db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
|
||||
db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -523,3 +564,11 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) maxExpiresAtForMemoryCache(expiresAt int64) int64 {
|
||||
var maxTimestamp = fasttime.Now().Unix() + 3600
|
||||
if expiresAt > maxTimestamp {
|
||||
return maxTimestamp
|
||||
}
|
||||
return expiresAt
|
||||
}
|
||||
|
||||
@@ -6,18 +6,15 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -29,8 +26,6 @@ type FileListDB struct {
|
||||
readDB *dbs.DB
|
||||
writeDB *dbs.DB
|
||||
|
||||
writeBatch *dbs.Batch
|
||||
|
||||
hashMap *FileListHashMap
|
||||
|
||||
itemsTableName string
|
||||
@@ -52,11 +47,10 @@ type FileListDB struct {
|
||||
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
deleteByHashSQL string
|
||||
|
||||
statStmt *dbs.Stmt // 统计
|
||||
purgeStmt *dbs.Stmt // 清理
|
||||
deleteAllStmt *dbs.Stmt // 删除所有数据
|
||||
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
|
||||
updateAccessWeekSQL string // 修改访问日期
|
||||
statStmt *dbs.Stmt // 统计
|
||||
purgeStmt *dbs.Stmt // 清理
|
||||
deleteAllStmt *dbs.Stmt // 删除所有数据
|
||||
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
|
||||
}
|
||||
|
||||
func NewFileListDB() *FileListDB {
|
||||
@@ -69,15 +63,15 @@ func (this *FileListDB) Open(dbPath string) error {
|
||||
this.dbPath = dbPath
|
||||
|
||||
// 动态调整Cache值
|
||||
var cacheSize = 32000
|
||||
var cacheSize = 512
|
||||
var memoryGB = utils.SystemMemoryGB()
|
||||
if memoryGB >= 8 {
|
||||
cacheSize += 32000 * memoryGB / 8
|
||||
if memoryGB >= 1 {
|
||||
cacheSize = 256 * memoryGB
|
||||
}
|
||||
|
||||
// write db
|
||||
// 这里不能加 EXCLUSIVE 锁,不然异步事务可能会失败
|
||||
writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
|
||||
writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
|
||||
if err != nil {
|
||||
return fmt.Errorf("open write database failed: %w", err)
|
||||
}
|
||||
@@ -104,17 +98,7 @@ func (this *FileListDB) Open(dbPath string) error {
|
||||
}
|
||||
}
|
||||
|
||||
this.writeBatch = dbs.NewBatch(writeDB, 4)
|
||||
this.writeBatch.OnFail(func(err error) {
|
||||
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error()+" ("+filepath.Base(this.dbPath)+")")
|
||||
})
|
||||
|
||||
goman.New(func() {
|
||||
this.writeBatch.Exec()
|
||||
})
|
||||
|
||||
if teaconst.EnableDBStat {
|
||||
this.writeBatch.EnableStat(true)
|
||||
this.writeDB.EnableStat(true)
|
||||
}
|
||||
|
||||
@@ -150,7 +134,7 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt", "accessWeek") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -161,7 +145,7 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
|
||||
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>? ORDER BY id ASC LIMIT 2000`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -187,13 +171,11 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
|
||||
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.updateAccessWeekSQL = `UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`
|
||||
|
||||
this.isReady = true
|
||||
|
||||
// 加载HashMap
|
||||
@@ -224,7 +206,7 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
item.StaleAt = item.ExpiredAt
|
||||
}
|
||||
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix(), timeutil.Format("YW"))
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, fasttime.Now().Unix())
|
||||
if err != nil {
|
||||
return this.WrapError(err)
|
||||
}
|
||||
@@ -320,8 +302,7 @@ func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHitAsync(hash string) error {
|
||||
var week = timeutil.Format("YW")
|
||||
this.writeBatch.Add(this.updateAccessWeekSQL, week, hash)
|
||||
// do nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -525,8 +506,7 @@ func (this *FileListDB) initTables(times int) error {
|
||||
"staleAt" integer DEFAULT 0,
|
||||
"createdAt" integer DEFAULT 0,
|
||||
"host" varchar(128),
|
||||
"serverId" integer,
|
||||
"accessWeek" varchar(6)
|
||||
"serverId" integer
|
||||
);
|
||||
|
||||
DROP INDEX IF EXISTS "createdAt";
|
||||
@@ -542,8 +522,6 @@ CREATE INDEX IF NOT EXISTS "hash"
|
||||
ON "` + this.itemsTableName + `" (
|
||||
"hash" ASC
|
||||
);
|
||||
|
||||
ALTER TABLE "cacheItems" ADD "accessWeek" varchar(6);
|
||||
`)
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -4,12 +4,20 @@ package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFileListDB_ListLFUItems(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
|
||||
defer func() {
|
||||
@@ -34,6 +42,10 @@ func TestFileListDB_ListLFUItems(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileListDB_CleanMatchKey(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
|
||||
defer func() {
|
||||
@@ -62,6 +74,10 @@ func TestFileListDB_CleanMatchKey(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileListDB_CleanMatchPrefix(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
|
||||
defer func() {
|
||||
@@ -88,3 +104,67 @@ func TestFileListDB_CleanMatchPrefix(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileListDB_Memory(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var db = caches.NewFileListDB()
|
||||
|
||||
defer func() {
|
||||
_ = db.Close()
|
||||
}()
|
||||
|
||||
err := db.Open(Tea.Root + "/data/cache-index/p1/db-0.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = db.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log(db.Total())
|
||||
|
||||
// load hashes
|
||||
var maxId int64
|
||||
var hashList []string
|
||||
var before = time.Now()
|
||||
for i := 0; i < 1_000; i++ {
|
||||
hashList, maxId, err = db.ListHashes(maxId)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hashList) == 0 {
|
||||
t.Log("hashes loaded", time.Since(before).Seconds()*1000, "ms")
|
||||
break
|
||||
}
|
||||
if i%100 == 0 {
|
||||
t.Log(i)
|
||||
}
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
|
||||
//time.Sleep(600 * time.Second)
|
||||
|
||||
for i := 0; i < 1_000; i++ {
|
||||
_, err = db.ListLFUItems(5000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i%100 == 0 {
|
||||
t.Log(i)
|
||||
}
|
||||
}
|
||||
|
||||
t.Log("loaded")
|
||||
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
|
||||
time.Sleep(600 * time.Second)
|
||||
}
|
||||
|
||||
@@ -9,18 +9,36 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
const HashMapSharding = 31
|
||||
|
||||
var bigIntPool = sync.Pool{
|
||||
New: func() any {
|
||||
return big.NewInt(0)
|
||||
},
|
||||
}
|
||||
|
||||
// FileListHashMap 文件Hash列表
|
||||
type FileListHashMap struct {
|
||||
m map[uint64]zero.Zero
|
||||
m []map[uint64]zero.Zero
|
||||
|
||||
lockers []*sync.RWMutex
|
||||
|
||||
locker sync.RWMutex
|
||||
isAvailable bool
|
||||
isReady bool
|
||||
}
|
||||
|
||||
func NewFileListHashMap() *FileListHashMap {
|
||||
var m = make([]map[uint64]zero.Zero, HashMapSharding)
|
||||
var lockers = make([]*sync.RWMutex, HashMapSharding)
|
||||
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
m[i] = map[uint64]zero.Zero{}
|
||||
lockers[i] = &sync.RWMutex{}
|
||||
}
|
||||
|
||||
return &FileListHashMap{
|
||||
m: map[uint64]zero.Zero{},
|
||||
m: m,
|
||||
lockers: lockers,
|
||||
isAvailable: false,
|
||||
isReady: false,
|
||||
}
|
||||
@@ -35,6 +53,7 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
this.isAvailable = true
|
||||
|
||||
var lastId int64
|
||||
var maxLoops = 50_000
|
||||
for {
|
||||
hashList, maxId, err := db.ListHashes(lastId)
|
||||
if err != nil {
|
||||
@@ -45,6 +64,11 @@ func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
}
|
||||
this.AddHashes(hashList)
|
||||
lastId = maxId
|
||||
|
||||
maxLoops--
|
||||
if maxLoops <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
this.isReady = true
|
||||
@@ -56,9 +80,11 @@ func (this *FileListHashMap) Add(hash string) {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
this.m[this.bigInt(hash)] = zero.New()
|
||||
this.locker.Unlock()
|
||||
hashInt, index := this.bigInt(hash)
|
||||
|
||||
this.lockers[index].Lock()
|
||||
this.m[index][hashInt] = zero.New()
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
@@ -66,11 +92,12 @@ func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
for _, hash := range hashes {
|
||||
this.m[this.bigInt(hash)] = zero.New()
|
||||
hashInt, index := this.bigInt(hash)
|
||||
this.lockers[index].Lock()
|
||||
this.m[index][hashInt] = zero.New()
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Delete(hash string) {
|
||||
@@ -78,9 +105,10 @@ func (this *FileListHashMap) Delete(hash string) {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
delete(this.m, this.bigInt(hash))
|
||||
this.locker.Unlock()
|
||||
hashInt, index := this.bigInt(hash)
|
||||
this.lockers[index].Lock()
|
||||
delete(this.m[index], hashInt)
|
||||
this.lockers[index].Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Exist(hash string) bool {
|
||||
@@ -91,16 +119,28 @@ func (this *FileListHashMap) Exist(hash string) bool {
|
||||
// 只有完全Ready时才能判断是否为false
|
||||
return true
|
||||
}
|
||||
this.locker.RLock()
|
||||
_, ok := this.m[this.bigInt(hash)]
|
||||
this.locker.RUnlock()
|
||||
|
||||
hashInt, index := this.bigInt(hash)
|
||||
|
||||
this.lockers[index].RLock()
|
||||
_, ok := this.m[index][hashInt]
|
||||
this.lockers[index].RUnlock()
|
||||
return ok
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Clean() {
|
||||
this.locker.Lock()
|
||||
this.m = map[uint64]zero.Zero{}
|
||||
this.locker.Unlock()
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.lockers[i].Lock()
|
||||
}
|
||||
|
||||
// 这里不能简单清空 this.m ,避免导致别的数据无法写入 map 而产生 panic
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.m[i] = map[uint64]zero.Zero{}
|
||||
}
|
||||
|
||||
for i := HashMapSharding - 1; i >= 0; i-- {
|
||||
this.lockers[i].Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) IsReady() bool {
|
||||
@@ -108,13 +148,36 @@ func (this *FileListHashMap) IsReady() bool {
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Len() int {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
return len(this.m)
|
||||
for i := 0; i < HashMapSharding; i++ {
|
||||
this.lockers[i].Lock()
|
||||
}
|
||||
|
||||
var count = 0
|
||||
for _, shard := range this.m {
|
||||
count += len(shard)
|
||||
}
|
||||
|
||||
for i := HashMapSharding - 1; i >= 0; i-- {
|
||||
this.lockers[i].Unlock()
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) bigInt(hash string) uint64 {
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
return bigInt.Uint64()
|
||||
func (this *FileListHashMap) SetIsAvailable(isAvailable bool) {
|
||||
this.isAvailable = isAvailable
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) SetIsReady(isReady bool) {
|
||||
this.isReady = isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) bigInt(hash string) (hashInt uint64, index int) {
|
||||
var bigInt = bigIntPool.Get().(*big.Int)
|
||||
bigInt.SetString(hash, 16)
|
||||
hashInt = bigInt.Uint64()
|
||||
bigIntPool.Put(bigInt)
|
||||
|
||||
index = int(hashInt % HashMapSharding)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"math/big"
|
||||
@@ -20,15 +22,19 @@ func TestFileListHashMap_Memory(t *testing.T) {
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m.Add(stringutil.Md5(types.String(i)))
|
||||
}
|
||||
|
||||
t.Log("added:", m.Len(), "hashes")
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
|
||||
t.Log("remains:", m.Len(), "hashes")
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Memory2(t *testing.T) {
|
||||
@@ -48,12 +54,34 @@ func TestFileListHashMap_Memory2(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileListHashMap_BigInt(t *testing.T) {
|
||||
var bigInt = big.NewInt(0)
|
||||
|
||||
for _, s := range []string{"1", "2", "3", "123", "123456"} {
|
||||
var hash = stringutil.Md5(s)
|
||||
|
||||
var bigInt = big.NewInt(0)
|
||||
var bigInt1 = big.NewInt(0)
|
||||
bigInt1.SetString(hash, 16)
|
||||
|
||||
bigInt.SetString(hash, 16)
|
||||
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
|
||||
|
||||
t.Log(s, "=>", bigInt1.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt1.Uint64(), 16), strconv.FormatUint(bigInt.Uint64(), 16))
|
||||
|
||||
if strconv.FormatUint(bigInt1.Uint64(), 16) != strconv.FormatUint(bigInt.Uint64(), 16) {
|
||||
t.Fatal("not equal")
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
var hash = stringutil.Md5(types.String(i))
|
||||
|
||||
var bigInt1 = big.NewInt(0)
|
||||
bigInt1.SetString(hash, 16)
|
||||
|
||||
bigInt.SetString(hash, 16)
|
||||
|
||||
if bigInt1.Uint64() != bigInt.Uint64() {
|
||||
t.Fatal(i, "not equal")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +113,25 @@ func TestFileListHashMap_Load(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Delete(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
m.SetIsReady(true)
|
||||
m.SetIsAvailable(true)
|
||||
m.Add("a")
|
||||
a.IsTrue(m.Len() == 1)
|
||||
m.Delete("a")
|
||||
a.IsTrue(m.Len() == 0)
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Clean(t *testing.T) {
|
||||
var m = caches.NewFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
m.Clean()
|
||||
m.Add("a")
|
||||
}
|
||||
|
||||
func Benchmark_BigInt(b *testing.B) {
|
||||
var hash = stringutil.Md5("123456")
|
||||
b.ResetTimer()
|
||||
@@ -95,3 +142,23 @@ func Benchmark_BigInt(b *testing.B) {
|
||||
_ = bigInt.Uint64()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFileListHashMap_Exist(b *testing.B) {
|
||||
var m = caches.NewFileListHashMap()
|
||||
m.SetIsAvailable(true)
|
||||
m.SetIsReady(true)
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m.Add(types.String(i))
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
m.Add(types.String(rands.Int64()))
|
||||
_ = m.Exist(types.String(rands.Int64()))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"golang.org/x/sys/unix"
|
||||
@@ -35,6 +36,9 @@ type Manager struct {
|
||||
SubDiskDirs []*serverconfigs.CacheDir
|
||||
MaxMemoryCapacity *shared.SizeCapacity
|
||||
|
||||
CountFileStorages int
|
||||
CountMemoryStorages int
|
||||
|
||||
policyMap map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
|
||||
storageMap map[int64]StorageInterface // policyId => *Storage
|
||||
locker sync.RWMutex
|
||||
@@ -143,6 +147,16 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.CountFileStorages = 0
|
||||
this.CountFileStorages = 0
|
||||
for _, storage := range this.storageMap {
|
||||
_, isFileStorage := storage.(*FileStorage)
|
||||
this.CountMemoryStorages++
|
||||
if isFileStorage {
|
||||
this.CountFileStorages++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FindPolicy 获取Policy信息
|
||||
@@ -172,6 +186,11 @@ func (this *Manager) NewStorageWithPolicy(policy *serverconfigs.HTTPCachePolicy)
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageMap 获取已有的存储对象
|
||||
func (this *Manager) StorageMap() map[int64]StorageInterface {
|
||||
return this.storageMap
|
||||
}
|
||||
|
||||
// TotalDiskSize 消耗的磁盘尺寸
|
||||
func (this *Manager) TotalDiskSize() int64 {
|
||||
this.locker.RLock()
|
||||
@@ -272,3 +291,17 @@ func (this *Manager) ScanGarbageCaches(callback func(path string) error) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MaxSystemMemoryBytesPerStorage 计算单个策略能使用的系统最大内存
|
||||
func (this *Manager) MaxSystemMemoryBytesPerStorage() int64 {
|
||||
var count = this.CountMemoryStorages
|
||||
if count < 1 {
|
||||
count = 1
|
||||
}
|
||||
|
||||
var resultBytes = int64(utils.SystemMemoryBytes()) / 3 / int64(count) // 1/3 of the system memory
|
||||
if resultBytes < 1<<30 {
|
||||
resultBytes = 1 << 30
|
||||
}
|
||||
return resultBytes
|
||||
}
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
package caches
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"testing"
|
||||
)
|
||||
@@ -10,7 +11,7 @@ import (
|
||||
func TestManager_UpdatePolicies(t *testing.T) {
|
||||
{
|
||||
var policies = []*serverconfigs.HTTPCachePolicy{}
|
||||
SharedManager.UpdatePolicies(policies)
|
||||
caches.SharedManager.UpdatePolicies(policies)
|
||||
printManager(t)
|
||||
}
|
||||
|
||||
@@ -38,7 +39,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
SharedManager.UpdatePolicies(policies)
|
||||
caches.SharedManager.UpdatePolicies(policies)
|
||||
printManager(t)
|
||||
}
|
||||
|
||||
@@ -66,7 +67,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
SharedManager.UpdatePolicies(policies)
|
||||
caches.SharedManager.UpdatePolicies(policies)
|
||||
printManager(t)
|
||||
}
|
||||
}
|
||||
@@ -80,8 +81,8 @@ func TestManager_ChangePolicy_Memory(t *testing.T) {
|
||||
Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
|
||||
},
|
||||
}
|
||||
SharedManager.UpdatePolicies(policies)
|
||||
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
|
||||
caches.SharedManager.UpdatePolicies(policies)
|
||||
caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
|
||||
{
|
||||
Id: 1,
|
||||
Type: serverconfigs.CachePolicyStorageMemory,
|
||||
@@ -102,8 +103,8 @@ func TestManager_ChangePolicy_File(t *testing.T) {
|
||||
Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
|
||||
},
|
||||
}
|
||||
SharedManager.UpdatePolicies(policies)
|
||||
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
|
||||
caches.SharedManager.UpdatePolicies(policies)
|
||||
caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
|
||||
{
|
||||
Id: 1,
|
||||
Type: serverconfigs.CachePolicyStorageFile,
|
||||
@@ -115,10 +116,17 @@ func TestManager_ChangePolicy_File(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestManager_MaxSystemMemoryBytesPerStorage(t *testing.T) {
|
||||
for i := 0; i < 100; i++ {
|
||||
caches.SharedManager.CountMemoryStorages = i
|
||||
t.Log(i, caches.SharedManager.MaxSystemMemoryBytesPerStorage()>>30, "GB")
|
||||
}
|
||||
}
|
||||
|
||||
func printManager(t *testing.T) {
|
||||
t.Log("===manager==")
|
||||
t.Log("storage:")
|
||||
for _, storage := range SharedManager.storageMap {
|
||||
for _, storage := range caches.SharedManager.StorageMap() {
|
||||
t.Log(" storage:", storage.Policy().Id)
|
||||
}
|
||||
t.Log("===============")
|
||||
|
||||
375
internal/caches/memory_fragment_pool.go
Normal file
375
internal/caches/memory_fragment_pool.go
Normal file
@@ -0,0 +1,375 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
enableFragmentPool = false
|
||||
minMemoryFragmentPoolItemSize = 8 << 10
|
||||
maxMemoryFragmentPoolItemSize = 128 << 20
|
||||
maxItemsInMemoryFragmentPoolBucket = 1024
|
||||
memoryFragmentPoolBucketSegmentSize = 512 << 10
|
||||
maxMemoryFragmentPoolItemAgeSeconds = 60
|
||||
)
|
||||
|
||||
var SharedFragmentMemoryPool *MemoryFragmentPool
|
||||
|
||||
func init() {
|
||||
if !teaconst.IsMain {
|
||||
return
|
||||
}
|
||||
|
||||
SharedFragmentMemoryPool = NewMemoryFragmentPool()
|
||||
|
||||
goman.New(func() {
|
||||
var ticker = time.NewTicker(200 * time.Millisecond)
|
||||
for range ticker.C {
|
||||
for i := 0; i < 10; i++ { // skip N empty buckets
|
||||
var isEmpty = SharedFragmentMemoryPool.GCNextBucket()
|
||||
if !isEmpty {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type MemoryFragmentPoolItem struct {
|
||||
Bytes []byte
|
||||
|
||||
size int64
|
||||
createdAt int64
|
||||
|
||||
Refs int32
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPoolItem) IsExpired() bool {
|
||||
return this.createdAt < fasttime.Now().Unix()-maxMemoryFragmentPoolItemAgeSeconds
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPoolItem) Reset() {
|
||||
this.Bytes = nil
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPoolItem) IsAvailable() bool {
|
||||
return atomic.AddInt32(&this.Refs, 1) == 1
|
||||
}
|
||||
|
||||
// MemoryFragmentPool memory fragments management
|
||||
type MemoryFragmentPool struct {
|
||||
bucketMaps []map[uint64]*MemoryFragmentPoolItem // [ { id => Zero }, ... ]
|
||||
countBuckets int
|
||||
gcBucketIndex int
|
||||
|
||||
mu sync.RWMutex
|
||||
|
||||
id uint64
|
||||
totalMemory int64
|
||||
|
||||
isOk bool
|
||||
capacity int64
|
||||
|
||||
debugMode bool
|
||||
countGet uint64
|
||||
countNew uint64
|
||||
}
|
||||
|
||||
// NewMemoryFragmentPool create new fragment memory pool
|
||||
func NewMemoryFragmentPool() *MemoryFragmentPool {
|
||||
var pool = &MemoryFragmentPool{}
|
||||
pool.init()
|
||||
return pool
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) init() {
|
||||
var capacity = int64(utils.SystemMemoryGB()) << 30 / 16
|
||||
if capacity > 256<<20 {
|
||||
this.isOk = true
|
||||
this.capacity = capacity
|
||||
|
||||
this.bucketMaps = []map[uint64]*MemoryFragmentPoolItem{}
|
||||
for i := 0; i < maxMemoryFragmentPoolItemSize/memoryFragmentPoolBucketSegmentSize+1; i++ {
|
||||
this.bucketMaps = append(this.bucketMaps, map[uint64]*MemoryFragmentPoolItem{})
|
||||
}
|
||||
this.countBuckets = len(this.bucketMaps)
|
||||
}
|
||||
|
||||
// print statistics for debug
|
||||
if len(os.Getenv("GOEDGE_DEBUG_MEMORY_FRAGMENT_POOL")) > 0 {
|
||||
this.debugMode = true
|
||||
|
||||
go func() {
|
||||
var maxRounds = 10_000
|
||||
var ticker = time.NewTicker(10 * time.Second)
|
||||
for range ticker.C {
|
||||
logs.Println("reused:", this.countGet, "created:", this.countNew, "fragments:", this.Len(), "memory:", this.totalMemory>>20, "MB")
|
||||
|
||||
maxRounds--
|
||||
if maxRounds <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// Get try to get a bytes object
|
||||
func (this *MemoryFragmentPool) Get(expectSize int64) (resultBytes []byte, ok bool) {
|
||||
if !this.isOk {
|
||||
return
|
||||
}
|
||||
|
||||
if expectSize <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// DO NOT check min segment size
|
||||
|
||||
this.mu.RLock()
|
||||
|
||||
var bucketIndex = this.bucketIndexForSize(expectSize)
|
||||
var resultItemId uint64
|
||||
const maxSearchingBuckets = 20
|
||||
for i := bucketIndex; i <= bucketIndex+maxSearchingBuckets; i++ {
|
||||
resultBytes, resultItemId, ok = this.findItemInMap(this.bucketMaps[i], expectSize)
|
||||
if ok {
|
||||
this.mu.RUnlock()
|
||||
|
||||
// remove from bucket
|
||||
this.mu.Lock()
|
||||
delete(this.bucketMaps[i], resultItemId)
|
||||
this.mu.Unlock()
|
||||
|
||||
return
|
||||
}
|
||||
if i >= this.countBuckets {
|
||||
break
|
||||
}
|
||||
}
|
||||
this.mu.RUnlock()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Put a bytes object to specified bucket
|
||||
func (this *MemoryFragmentPool) Put(data []byte) (ok bool) {
|
||||
if !this.isOk {
|
||||
return
|
||||
}
|
||||
|
||||
var l = int64(cap(data)) // MUST be 'cap' instead of 'len'
|
||||
|
||||
if l < minMemoryFragmentPoolItemSize || l > maxMemoryFragmentPoolItemSize {
|
||||
return
|
||||
}
|
||||
|
||||
if atomic.LoadInt64(&this.totalMemory) >= this.capacity {
|
||||
return
|
||||
}
|
||||
|
||||
var itemId = atomic.AddUint64(&this.id, 1)
|
||||
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
var bucketMap = this.bucketMaps[this.bucketIndexForSize(l)]
|
||||
if len(bucketMap) >= maxItemsInMemoryFragmentPoolBucket {
|
||||
return
|
||||
}
|
||||
|
||||
atomic.AddInt64(&this.totalMemory, l)
|
||||
|
||||
bucketMap[itemId] = &MemoryFragmentPoolItem{
|
||||
Bytes: data,
|
||||
size: l,
|
||||
createdAt: fasttime.Now().Unix(),
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// GC fully GC
|
||||
func (this *MemoryFragmentPool) GC() {
|
||||
if !this.isOk {
|
||||
return
|
||||
}
|
||||
|
||||
var totalMemory = atomic.LoadInt64(&this.totalMemory)
|
||||
if totalMemory < this.capacity {
|
||||
return
|
||||
}
|
||||
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
var garbageSize = totalMemory * 1 / 10 // 10%
|
||||
|
||||
// remove expired
|
||||
for _, bucketMap := range this.bucketMaps {
|
||||
for itemId, item := range bucketMap {
|
||||
if item.IsExpired() {
|
||||
delete(bucketMap, itemId)
|
||||
item.Reset()
|
||||
atomic.AddInt64(&this.totalMemory, -item.size)
|
||||
|
||||
garbageSize -= item.size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove others
|
||||
if garbageSize > 0 {
|
||||
for _, bucketMap := range this.bucketMaps {
|
||||
for itemId, item := range bucketMap {
|
||||
delete(bucketMap, itemId)
|
||||
item.Reset()
|
||||
atomic.AddInt64(&this.totalMemory, -item.size)
|
||||
|
||||
garbageSize -= item.size
|
||||
if garbageSize <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GCNextBucket gc one bucket
|
||||
func (this *MemoryFragmentPool) GCNextBucket() (isEmpty bool) {
|
||||
if !this.isOk {
|
||||
return
|
||||
}
|
||||
|
||||
var itemIds = []uint64{}
|
||||
|
||||
// find
|
||||
this.mu.RLock()
|
||||
|
||||
var bucketIndex = this.gcBucketIndex
|
||||
var bucketMap = this.bucketMaps[bucketIndex]
|
||||
isEmpty = len(bucketMap) == 0
|
||||
if isEmpty {
|
||||
this.mu.RUnlock()
|
||||
|
||||
// move to next bucket index
|
||||
bucketIndex++
|
||||
if bucketIndex >= this.countBuckets {
|
||||
bucketIndex = 0
|
||||
}
|
||||
this.gcBucketIndex = bucketIndex
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for itemId, item := range bucketMap {
|
||||
if item.IsExpired() {
|
||||
itemIds = append(itemIds, itemId)
|
||||
}
|
||||
}
|
||||
|
||||
this.mu.RUnlock()
|
||||
|
||||
// remove
|
||||
if len(itemIds) > 0 {
|
||||
this.mu.Lock()
|
||||
for _, itemId := range itemIds {
|
||||
item, ok := bucketMap[itemId]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !item.IsAvailable() {
|
||||
continue
|
||||
}
|
||||
delete(bucketMap, itemId)
|
||||
item.Reset()
|
||||
atomic.AddInt64(&this.totalMemory, -item.size)
|
||||
}
|
||||
this.mu.Unlock()
|
||||
}
|
||||
|
||||
// move to next bucket index
|
||||
bucketIndex++
|
||||
if bucketIndex >= this.countBuckets {
|
||||
bucketIndex = 0
|
||||
}
|
||||
this.gcBucketIndex = bucketIndex
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) SetCapacity(capacity int64) {
|
||||
this.capacity = capacity
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) TotalSize() int64 {
|
||||
return atomic.LoadInt64(&this.totalMemory)
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) Len() int {
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
var count = 0
|
||||
for _, bucketMap := range this.bucketMaps {
|
||||
count += len(bucketMap)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) IncreaseNew() {
|
||||
if this.isOk && this.debugMode {
|
||||
atomic.AddUint64(&this.countNew, 1)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) bucketIndexForSize(size int64) int {
|
||||
return int(size / memoryFragmentPoolBucketSegmentSize)
|
||||
}
|
||||
|
||||
func (this *MemoryFragmentPool) findItemInMap(bucketMap map[uint64]*MemoryFragmentPoolItem, expectSize int64) (resultBytes []byte, resultItemId uint64, ok bool) {
|
||||
if len(bucketMap) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for itemId, item := range bucketMap {
|
||||
if item.size >= expectSize {
|
||||
// check if is referred
|
||||
if !item.IsAvailable() {
|
||||
continue
|
||||
}
|
||||
|
||||
// return result
|
||||
if item.size != expectSize {
|
||||
resultBytes = item.Bytes[:expectSize]
|
||||
} else {
|
||||
resultBytes = item.Bytes
|
||||
}
|
||||
|
||||
// reset old item
|
||||
item.Reset()
|
||||
atomic.AddInt64(&this.totalMemory, -item.size)
|
||||
|
||||
resultItemId = itemId
|
||||
|
||||
if this.debugMode {
|
||||
atomic.AddUint64(&this.countGet, 1)
|
||||
}
|
||||
|
||||
ok = true
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
313
internal/caches/memory_fragment_pool_test.go
Normal file
313
internal/caches/memory_fragment_pool_test.go
Normal file
@@ -0,0 +1,313 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewMemoryFragmentPool(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
for i := 0; i < 3000; i++ {
|
||||
ok := pool.Put(make([]byte, 2<<20))
|
||||
if !ok {
|
||||
t.Log("finished at", i)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
t.Log(pool.TotalSize()>>20, "MB", pool.Len(), "items")
|
||||
|
||||
{
|
||||
r, ok := pool.Get(1 << 20)
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(r) == 1<<20)
|
||||
}
|
||||
|
||||
{
|
||||
r, ok := pool.Get(2 << 20)
|
||||
a.IsTrue(ok)
|
||||
a.IsTrue(len(r) == 2<<20)
|
||||
}
|
||||
|
||||
{
|
||||
r, ok := pool.Get(4 << 20)
|
||||
a.IsFalse(ok)
|
||||
a.IsTrue(len(r) == 0)
|
||||
}
|
||||
|
||||
t.Log(pool.TotalSize()>>20, "MB", pool.Len(), "items")
|
||||
}
|
||||
|
||||
func TestNewMemoryFragmentPool_LargeBucket(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
{
|
||||
pool.Put(make([]byte, 128<<20+1))
|
||||
a.IsTrue(pool.Len() == 0)
|
||||
}
|
||||
|
||||
{
|
||||
pool.Put(make([]byte, 128<<20))
|
||||
a.IsTrue(pool.Len() == 1)
|
||||
|
||||
pool.Get(118 << 20)
|
||||
a.IsTrue(pool.Len() == 0)
|
||||
}
|
||||
|
||||
{
|
||||
pool.Put(make([]byte, 128<<20))
|
||||
a.IsTrue(pool.Len() == 1)
|
||||
|
||||
pool.Get(110 << 20)
|
||||
a.IsTrue(pool.Len() == 1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryFragmentPool_Get_Exactly(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
{
|
||||
pool.Put(make([]byte, 129<<20))
|
||||
a.IsTrue(pool.Len() == 0)
|
||||
}
|
||||
|
||||
{
|
||||
pool.Put(make([]byte, 4<<20))
|
||||
a.IsTrue(pool.Len() == 1)
|
||||
}
|
||||
|
||||
{
|
||||
pool.Get(4 << 20)
|
||||
a.IsTrue(pool.Len() == 0)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryFragmentPool_Get_Round(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
{
|
||||
pool.Put(make([]byte, 8<<20))
|
||||
pool.Put(make([]byte, 8<<20))
|
||||
pool.Put(make([]byte, 8<<20))
|
||||
a.IsTrue(pool.Len() == 3)
|
||||
}
|
||||
|
||||
{
|
||||
resultBytes, ok := pool.Get(3 << 20)
|
||||
a.IsTrue(pool.Len() == 2)
|
||||
if ok {
|
||||
pool.Put(resultBytes)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
pool.Get(2 << 20)
|
||||
a.IsTrue(pool.Len() == 2)
|
||||
}
|
||||
|
||||
{
|
||||
pool.Get(1 << 20)
|
||||
a.IsTrue(pool.Len() == 1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryFragmentPool_GC(t *testing.T) {
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
pool.SetCapacity(32 << 20)
|
||||
for i := 0; i < 16; i++ {
|
||||
pool.Put(make([]byte, 4<<20))
|
||||
}
|
||||
var before = time.Now()
|
||||
pool.GC()
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
t.Log(pool.Len())
|
||||
}
|
||||
|
||||
func TestMemoryFragmentPool_Memory(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
|
||||
testutils.StartMemoryStats(t, func() {
|
||||
t.Log(pool.Len(), "items")
|
||||
})
|
||||
|
||||
var sampleData = bytes.Repeat([]byte{'A'}, 16<<20)
|
||||
|
||||
var countNew = 0
|
||||
for i := 0; i < 1000; i++ {
|
||||
cacheData, ok := pool.Get(16 << 20)
|
||||
if ok {
|
||||
copy(cacheData, sampleData)
|
||||
pool.Put(cacheData)
|
||||
} else {
|
||||
countNew++
|
||||
var data = make([]byte, 16<<20)
|
||||
copy(data, sampleData)
|
||||
pool.Put(data)
|
||||
}
|
||||
}
|
||||
|
||||
t.Log("count new:", countNew)
|
||||
t.Log("count remains:", pool.Len())
|
||||
|
||||
time.Sleep(10 * time.Minute)
|
||||
}
|
||||
|
||||
func TestMemoryFragmentPool_GCNextBucket(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
for i := 0; i < 1000; i++ {
|
||||
pool.Put(make([]byte, rands.Int(0, 100)<<20))
|
||||
}
|
||||
|
||||
var lastLen int
|
||||
for {
|
||||
pool.GCNextBucket()
|
||||
var currentLen = pool.Len()
|
||||
if lastLen == currentLen {
|
||||
continue
|
||||
}
|
||||
lastLen = currentLen
|
||||
|
||||
t.Log(currentLen, "items", pool.TotalSize(), "bytes", timeutil.Format("H:i:s"))
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if currentLen == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryFragmentPoolItem(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var m = map[int]*caches.MemoryFragmentPoolItem{}
|
||||
m[1] = &caches.MemoryFragmentPoolItem{
|
||||
Refs: 0,
|
||||
}
|
||||
var item = m[1]
|
||||
a.IsTrue(item.Refs == 0)
|
||||
a.IsTrue(atomic.AddInt32(&item.Refs, 1) == 1)
|
||||
|
||||
for _, item2 := range m {
|
||||
t.Log(item2)
|
||||
a.IsTrue(atomic.AddInt32(&item2.Refs, 1) == 2)
|
||||
}
|
||||
|
||||
t.Log(m)
|
||||
}
|
||||
|
||||
func BenchmarkMemoryFragmentPool_Get_HIT(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
for i := 0; i < 3000; i++ {
|
||||
ok := pool.Put(make([]byte, 2<<20))
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
data, ok := pool.Get(2 << 20)
|
||||
if ok {
|
||||
pool.Put(data)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkMemoryFragmentPool_Get_TOTALLY_MISSING(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
for i := 0; i < 3000; i++ {
|
||||
ok := pool.Put(make([]byte, 2<<20+100))
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
data, ok := pool.Get(2<<20 + 200)
|
||||
if ok {
|
||||
pool.Put(data)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkMemoryPool_Get_HIT_MISSING(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
for i := 0; i < 3000; i++ {
|
||||
ok := pool.Put(make([]byte, rands.Int(2, 32)<<20))
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
data, ok := pool.Get(4 << 20)
|
||||
if ok {
|
||||
pool.Put(data)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkMemoryFragmentPool_GC(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var pool = caches.NewMemoryFragmentPool()
|
||||
pool.SetCapacity(1 << 30)
|
||||
for i := 0; i < 2_000; i++ {
|
||||
pool.Put(make([]byte, 1<<20))
|
||||
}
|
||||
|
||||
var mu = sync.Mutex{}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
mu.Lock()
|
||||
for i := 0; i < 100; i++ {
|
||||
pool.GCNextBucket()
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -12,14 +12,16 @@ type OpenFile struct {
|
||||
meta []byte
|
||||
header []byte
|
||||
version int64
|
||||
size int64
|
||||
}
|
||||
|
||||
func NewOpenFile(fp *os.File, meta []byte, header []byte, version int64) *OpenFile {
|
||||
func NewOpenFile(fp *os.File, meta []byte, header []byte, version int64, size int64) *OpenFile {
|
||||
return &OpenFile{
|
||||
fp: fp,
|
||||
meta: meta,
|
||||
header: header,
|
||||
version: version,
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
package caches
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
@@ -14,26 +16,34 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxOpenFileSize = 256 << 20
|
||||
)
|
||||
|
||||
type OpenFileCache struct {
|
||||
poolMap map[string]*OpenFilePool // file path => Pool
|
||||
poolList *linkedlist.List
|
||||
poolList *linkedlist.List[*OpenFilePool]
|
||||
watcher *fsnotify.Watcher
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
maxSize int
|
||||
count int
|
||||
maxCount int
|
||||
capacitySize int64
|
||||
|
||||
count int
|
||||
usedSize int64
|
||||
}
|
||||
|
||||
func NewOpenFileCache(maxSize int) (*OpenFileCache, error) {
|
||||
if maxSize <= 0 {
|
||||
maxSize = 16384
|
||||
func NewOpenFileCache(maxCount int) (*OpenFileCache, error) {
|
||||
if maxCount <= 0 {
|
||||
maxCount = 16384
|
||||
}
|
||||
|
||||
var cache = &OpenFileCache{
|
||||
maxSize: maxSize,
|
||||
poolMap: map[string]*OpenFilePool{},
|
||||
poolList: linkedlist.NewList(),
|
||||
maxCount: maxCount,
|
||||
poolMap: map[string]*OpenFilePool{},
|
||||
poolList: linkedlist.NewList[*OpenFilePool](),
|
||||
capacitySize: (int64(utils.SystemMemoryGB()) << 30) / 16,
|
||||
}
|
||||
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
@@ -58,24 +68,36 @@ func (this *OpenFileCache) Get(filename string) *OpenFile {
|
||||
pool, ok := this.poolMap[filename]
|
||||
this.locker.RUnlock()
|
||||
if ok {
|
||||
file, consumed := pool.Get()
|
||||
file, consumed, consumedSize := pool.Get()
|
||||
if consumed {
|
||||
this.locker.Lock()
|
||||
this.count--
|
||||
this.usedSize -= consumedSize
|
||||
|
||||
// pool如果为空,也不需要从列表中删除,避免put时需要重新创建
|
||||
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) Put(filename string, file *OpenFile) {
|
||||
if file.size > maxOpenFileSize {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
// 如果超过当前容量,则关闭最早的
|
||||
if this.count >= this.maxCount || this.usedSize+file.size >= this.capacitySize {
|
||||
this.consumeHead()
|
||||
return
|
||||
}
|
||||
|
||||
pool, ok := this.poolMap[filename]
|
||||
var success bool
|
||||
if ok {
|
||||
@@ -92,35 +114,7 @@ func (this *OpenFileCache) Put(filename string, file *OpenFile) {
|
||||
// 检查长度
|
||||
if success {
|
||||
this.count++
|
||||
|
||||
// 如果超过当前容量,则关闭最早的
|
||||
if this.count > this.maxSize {
|
||||
var delta = this.maxSize / 100 // 清理1%
|
||||
if delta == 0 {
|
||||
delta = 1
|
||||
}
|
||||
for i := 0; i < delta; i++ {
|
||||
var head = this.poolList.Head()
|
||||
if head == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var headPool = head.Value.(*OpenFilePool)
|
||||
headFile, consumed := headPool.Get()
|
||||
if consumed {
|
||||
this.count--
|
||||
if headFile != nil {
|
||||
_ = headFile.Close()
|
||||
}
|
||||
}
|
||||
|
||||
if headPool.Len() == 0 {
|
||||
delete(this.poolMap, headPool.filename)
|
||||
this.poolList.Remove(head)
|
||||
_ = this.watcher.Remove(headPool.filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
this.usedSize += file.size
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,6 +130,7 @@ func (this *OpenFileCache) Close(filename string) {
|
||||
this.poolList.Remove(pool.linkItem)
|
||||
_ = this.watcher.Remove(filename)
|
||||
this.count -= pool.Len()
|
||||
this.usedSize -= pool.usedSize
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
@@ -155,18 +150,55 @@ func (this *OpenFileCache) CloseAll() {
|
||||
this.poolList.Reset()
|
||||
_ = this.watcher.Close()
|
||||
this.count = 0
|
||||
this.usedSize = 0
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) SetCapacity(capacityBytes int64) {
|
||||
this.capacitySize = capacityBytes
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) Debug() {
|
||||
var ticker = time.NewTicker(5 * time.Second)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
logs.Println("==== " + types.String(this.count) + " ====")
|
||||
this.poolList.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
logs.Println(filepath.Base(item.Value.(*OpenFilePool).Filename()), item.Value.(*OpenFilePool).Len())
|
||||
logs.Println("==== " + types.String(this.count) + ", " + fmt.Sprintf("%.4fMB", float64(this.usedSize)/(1<<20)) + " ====")
|
||||
this.poolList.Range(func(item *linkedlist.Item[*OpenFilePool]) (goNext bool) {
|
||||
logs.Println(filepath.Base(item.Value.Filename()), item.Value.Len())
|
||||
return true
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) consumeHead() {
|
||||
var delta = 1
|
||||
|
||||
if this.count > 100 {
|
||||
delta = 2
|
||||
}
|
||||
|
||||
for i := 0; i < delta; i++ {
|
||||
var head = this.poolList.Head()
|
||||
if head == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var headPool = head.Value
|
||||
headFile, consumed, consumedSize := headPool.Get()
|
||||
if consumed {
|
||||
this.count--
|
||||
this.usedSize -= consumedSize
|
||||
|
||||
if headFile != nil {
|
||||
_ = headFile.Close()
|
||||
}
|
||||
}
|
||||
|
||||
if headPool.Len() == 0 {
|
||||
delete(this.poolMap, headPool.filename)
|
||||
this.poolList.Remove(head)
|
||||
_ = this.watcher.Remove(headPool.filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ package caches_test
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -15,13 +16,14 @@ func TestNewOpenFileCache_Close(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache.Debug()
|
||||
cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
|
||||
cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0, 1<<20))
|
||||
|
||||
cache.Get("b.txt")
|
||||
cache.Get("d.txt")
|
||||
cache.Get("d.txt") // not exist
|
||||
cache.Close("a.txt")
|
||||
|
||||
if testutils.IsSingleTesting() {
|
||||
@@ -29,18 +31,39 @@ func TestNewOpenFileCache_Close(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOpenFileCache_OverSize(t *testing.T) {
|
||||
cache, err := caches.NewOpenFileCache(1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cache.SetCapacity(1 << 30)
|
||||
|
||||
cache.Debug()
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
cache.Put("a"+types.String(i)+".txt", caches.NewOpenFile(nil, nil, nil, 0, 128<<20))
|
||||
}
|
||||
|
||||
if testutils.IsSingleTesting() {
|
||||
time.Sleep(100 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOpenFileCache_CloseAll(t *testing.T) {
|
||||
cache, err := caches.NewOpenFileCache(1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache.Debug()
|
||||
cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
|
||||
cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0, 1))
|
||||
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0, 1))
|
||||
cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0, 1))
|
||||
cache.Get("b.txt")
|
||||
cache.Get("d.txt")
|
||||
cache.CloseAll()
|
||||
|
||||
time.Sleep(6 * time.Second)
|
||||
if testutils.IsSingleTesting() {
|
||||
time.Sleep(6 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,10 +9,11 @@ import (
|
||||
|
||||
type OpenFilePool struct {
|
||||
c chan *OpenFile
|
||||
linkItem *linkedlist.Item
|
||||
linkItem *linkedlist.Item[*OpenFilePool]
|
||||
filename string
|
||||
version int64
|
||||
isClosed bool
|
||||
usedSize int64
|
||||
}
|
||||
|
||||
func NewOpenFilePool(filename string) *OpenFilePool {
|
||||
@@ -21,7 +22,7 @@ func NewOpenFilePool(filename string) *OpenFilePool {
|
||||
c: make(chan *OpenFile, 1024),
|
||||
version: fasttime.Now().UnixMilli(),
|
||||
}
|
||||
pool.linkItem = linkedlist.NewItem(pool)
|
||||
pool.linkItem = linkedlist.NewItem[*OpenFilePool](pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
@@ -29,27 +30,29 @@ func (this *OpenFilePool) Filename() string {
|
||||
return this.filename
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) Get() (*OpenFile, bool) {
|
||||
func (this *OpenFilePool) Get() (resultFile *OpenFile, consumed bool, consumedSize int64) {
|
||||
// 如果已经关闭,直接返回
|
||||
if this.isClosed {
|
||||
return nil, false
|
||||
return nil, false, 0
|
||||
}
|
||||
|
||||
select {
|
||||
case file := <-this.c:
|
||||
if file != nil {
|
||||
this.usedSize -= file.size
|
||||
|
||||
err := file.SeekStart()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, true
|
||||
return nil, true, file.size
|
||||
}
|
||||
file.version = this.version
|
||||
|
||||
return file, true
|
||||
return file, true, file.size
|
||||
}
|
||||
return nil, false
|
||||
return nil, false, 0
|
||||
default:
|
||||
return nil, false
|
||||
return nil, false, 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +72,7 @@ func (this *OpenFilePool) Put(file *OpenFile) bool {
|
||||
// 加入Pool
|
||||
select {
|
||||
case this.c <- file:
|
||||
this.usedSize += file.size
|
||||
return true
|
||||
default:
|
||||
// 多余的直接关闭
|
||||
@@ -81,6 +85,10 @@ func (this *OpenFilePool) Len() int {
|
||||
return len(this.c)
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) TotalSize() int64 {
|
||||
return this.usedSize
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) SetClosing() {
|
||||
this.isClosed = true
|
||||
}
|
||||
|
||||
@@ -13,15 +13,15 @@ func TestOpenFilePool_Get(t *testing.T) {
|
||||
var pool = caches.NewOpenFilePool("a")
|
||||
t.Log(pool.Filename())
|
||||
t.Log(pool.Get())
|
||||
t.Log(pool.Put(caches.NewOpenFile(nil, nil, nil, 0)))
|
||||
t.Log(pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1)))
|
||||
t.Log(pool.Get())
|
||||
t.Log(pool.Get())
|
||||
}
|
||||
|
||||
func TestOpenFilePool_Close(t *testing.T) {
|
||||
var pool = caches.NewOpenFilePool("a")
|
||||
pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
|
||||
pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
|
||||
pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1))
|
||||
pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1))
|
||||
pool.Close()
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestOpenFilePool_Concurrent(t *testing.T) {
|
||||
defer wg.Done()
|
||||
|
||||
if rands.Int(0, 1) == 1 {
|
||||
pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
|
||||
pool.Put(caches.NewOpenFile(nil, nil, nil, 0, 1))
|
||||
}
|
||||
if rands.Int(0, 1) == 0 {
|
||||
pool.Get()
|
||||
|
||||
@@ -366,7 +366,7 @@ func (this *FileReader) Close() error {
|
||||
} else {
|
||||
var cacheMeta = make([]byte, len(this.meta))
|
||||
copy(cacheMeta, this.meta)
|
||||
this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, cacheMeta, this.header, this.LastModified()))
|
||||
this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, cacheMeta, this.header, this.LastModified(), this.bodySize))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/iwind/TeaGo/types"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"github.com/iwind/gosock/pkg/gosock"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
"math"
|
||||
"os"
|
||||
@@ -88,7 +89,8 @@ type FileStorage struct {
|
||||
|
||||
openFileCache *OpenFileCache
|
||||
|
||||
mainDiskIsFull bool
|
||||
mainDiskIsFull bool
|
||||
mainDiskTotalSize uint64
|
||||
|
||||
subDirs []*FileDir
|
||||
}
|
||||
@@ -421,6 +423,13 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
return nil, ErrEntityTooLarge
|
||||
}
|
||||
|
||||
// 检查磁盘是否超出容量
|
||||
// 需要在内存缓存之前执行,避免成功写进到了内存缓存,但无法刷到磁盘
|
||||
var capacityBytes = this.diskCapacityBytes()
|
||||
if capacityBytes > 0 && capacityBytes <= this.TotalDiskSize()+(32<<20 /** 余量 **/) {
|
||||
return nil, NewCapacityError("write file cache failed: over disk size, current: " + types.String(this.TotalDiskSize()) + ", capacity: " + types.String(capacityBytes))
|
||||
}
|
||||
|
||||
// 先尝试内存缓存
|
||||
// 我们限定仅小文件优先存在内存中
|
||||
var maxMemorySize = FileToMemoryMaxSize
|
||||
@@ -435,7 +444,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
}
|
||||
|
||||
// 如果队列满了,则等待
|
||||
if err == ErrWritingQueueFull {
|
||||
if errors.Is(err, ErrWritingQueueFull) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -464,12 +473,6 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
}
|
||||
}()
|
||||
|
||||
// 检查是否超出容量
|
||||
var capacityBytes = this.diskCapacityBytes()
|
||||
if capacityBytes > 0 && capacityBytes <= this.TotalDiskSize() {
|
||||
return nil, NewCapacityError("write file cache failed: over disk size, current total size: " + types.String(this.TotalDiskSize()) + " bytes, capacity: " + types.String(capacityBytes))
|
||||
}
|
||||
|
||||
var hash = stringutil.Md5(key)
|
||||
|
||||
dir, diskIsFull := this.subDir(hash)
|
||||
@@ -559,7 +562,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
if isNewCreated && existsFile {
|
||||
flags |= os.O_TRUNC
|
||||
}
|
||||
fsutils.WriteBegin()
|
||||
writer, err := os.OpenFile(tmpPath, flags, 0666)
|
||||
fsutils.WriteEnd()
|
||||
if err != nil {
|
||||
// TODO 检查在各个系统中的稳定性
|
||||
if os.IsNotExist(err) {
|
||||
@@ -997,18 +1002,26 @@ func (this *FileStorage) initList() error {
|
||||
// 清理任务
|
||||
// TODO purge每个分区
|
||||
func (this *FileStorage) purgeLoop() {
|
||||
// load
|
||||
systemLoad, _ := load.Avg()
|
||||
|
||||
// TODO 计算平均最近每日新增用量
|
||||
|
||||
// 计算是否应该开启LFU清理
|
||||
var capacityBytes = this.diskCapacityBytes()
|
||||
var startLFU = false
|
||||
var requireFullLFU = false // 是否需要完整执行LFU
|
||||
var lfuFreePercent = this.policy.PersistenceLFUFreePercent
|
||||
if lfuFreePercent <= 0 {
|
||||
lfuFreePercent = 5
|
||||
|
||||
// 2TB级别以上
|
||||
if capacityBytes>>30 > 2000 {
|
||||
lfuFreePercent = 100 /** GB **/ / float32(capacityBytes>>30) * 100 /** % **/
|
||||
if lfuFreePercent > 3 {
|
||||
lfuFreePercent = 3
|
||||
if systemLoad == nil || systemLoad.Load5 > 10 {
|
||||
// 2TB级别以上
|
||||
if capacityBytes>>30 > 2000 {
|
||||
lfuFreePercent = 100 /** GB **/ / float32(capacityBytes>>30) * 100 /** % **/
|
||||
if lfuFreePercent > 3 {
|
||||
lfuFreePercent = 3
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1032,30 +1045,45 @@ func (this *FileStorage) purgeLoop() {
|
||||
var times = 1
|
||||
|
||||
// 空闲时间多清理
|
||||
systemLoad, _ := load.Avg()
|
||||
if systemLoad != nil {
|
||||
if systemLoad.Load5 < 2 {
|
||||
if systemLoad.Load5 < 3 {
|
||||
times = 5
|
||||
} else if systemLoad.Load5 < 3 {
|
||||
times = 3
|
||||
} else if systemLoad.Load5 < 5 {
|
||||
times = 3
|
||||
} else if systemLoad.Load5 < 10 {
|
||||
times = 2
|
||||
}
|
||||
}
|
||||
|
||||
// 高速硬盘多清理
|
||||
if fsutils.DiskIsExtremelyFast() {
|
||||
times *= 8
|
||||
} else if fsutils.DiskIsFast() {
|
||||
times *= 4
|
||||
}
|
||||
|
||||
// 处于LFU阈值时,多清理
|
||||
if startLFU {
|
||||
times = 5
|
||||
times *= 5
|
||||
}
|
||||
|
||||
var purgeCount = this.policy.PersistenceAutoPurgeCount
|
||||
if purgeCount <= 0 {
|
||||
purgeCount = 1000
|
||||
|
||||
if fsutils.DiskIsExtremelyFast() {
|
||||
purgeCount = 4000
|
||||
} else if fsutils.DiskIsFast() {
|
||||
purgeCount = 2000
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < times; i++ {
|
||||
countFound, err := this.list.Purge(purgeCount, func(hash string) error {
|
||||
path, _ := this.hashPath(hash)
|
||||
fsutils.WriteBegin()
|
||||
err := this.removeCacheFile(path)
|
||||
fsutils.WriteEnd()
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
|
||||
}
|
||||
@@ -1068,6 +1096,10 @@ func (this *FileStorage) purgeLoop() {
|
||||
}
|
||||
|
||||
if countFound < purgeCount {
|
||||
if i == 0 && startLFU {
|
||||
requireFullLFU = true
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1077,13 +1109,13 @@ func (this *FileStorage) purgeLoop() {
|
||||
|
||||
// 磁盘空间不足时,清除老旧的缓存
|
||||
if startLFU {
|
||||
var maxCount = 2000
|
||||
var maxCount = 1000
|
||||
var maxLoops = 5
|
||||
|
||||
if fsutils.DiskIsFast() {
|
||||
maxCount = 5000
|
||||
} else if fsutils.DiskIsExtremelyFast() {
|
||||
maxCount = 10000
|
||||
if fsutils.DiskIsExtremelyFast() {
|
||||
maxCount = 4000
|
||||
} else if fsutils.DiskIsFast() {
|
||||
maxCount = 2000
|
||||
}
|
||||
|
||||
var total, _ = this.list.Count()
|
||||
@@ -1105,22 +1137,31 @@ func (this *FileStorage) purgeLoop() {
|
||||
count = maxCount
|
||||
}
|
||||
|
||||
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
|
||||
var before = time.Now()
|
||||
err := this.list.PurgeLFU(count, func(hash string) error {
|
||||
path, _ := this.hashPath(hash)
|
||||
fsutils.WriteBegin()
|
||||
err := this.removeCacheFile(path)
|
||||
fsutils.WriteEnd()
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
var prefix = ""
|
||||
if requireFullLFU {
|
||||
prefix = "fully "
|
||||
}
|
||||
remotelogs.Println("CACHE", prefix+"LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count)+", cost: "+fmt.Sprintf("%.2fms", time.Since(before).Seconds()*1000))
|
||||
|
||||
if err != nil {
|
||||
remotelogs.Warn("CACHE", "purge file storage in LFU failed: "+err.Error())
|
||||
}
|
||||
|
||||
// 检查硬盘空间状态
|
||||
if !this.hasFullDisk() {
|
||||
if !requireFullLFU && !this.hasFullDisk() {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1130,11 +1171,16 @@ func (this *FileStorage) purgeLoop() {
|
||||
|
||||
// 热点数据任务
|
||||
func (this *FileStorage) hotLoop() {
|
||||
var memoryStorage = this.memoryStorage
|
||||
var memoryStorage = this.memoryStorage // copy
|
||||
if memoryStorage == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// check memory space size
|
||||
if !memoryStorage.HasFreeSpaceForHotItems() {
|
||||
return
|
||||
}
|
||||
|
||||
this.hotMapLocker.Lock()
|
||||
if len(this.hotMap) == 0 {
|
||||
this.hotMapLocker.Unlock()
|
||||
@@ -1146,6 +1192,9 @@ func (this *FileStorage) hotLoop() {
|
||||
|
||||
var result = []*HotItem{} // [ {key: ..., hits: ...}, ... ]
|
||||
for _, v := range this.hotMap {
|
||||
if v.Hits <= 1 {
|
||||
continue
|
||||
}
|
||||
result = append(result, v)
|
||||
}
|
||||
|
||||
@@ -1253,9 +1302,17 @@ func (this *FileStorage) diskCapacityBytes() int64 {
|
||||
if nodeCapacity != nil {
|
||||
var c2 = nodeCapacity.Bytes()
|
||||
if c2 > 0 {
|
||||
if this.mainDiskTotalSize > 0 && c2 >= int64(this.mainDiskTotalSize) {
|
||||
c2 = int64(this.mainDiskTotalSize) * 95 / 100 // keep 5% free
|
||||
}
|
||||
return c2
|
||||
}
|
||||
}
|
||||
|
||||
if c1 <= 0 || (this.mainDiskTotalSize > 0 && c1 >= int64(this.mainDiskTotalSize)) {
|
||||
c1 = int64(this.mainDiskTotalSize) * 95 / 100 // keep 5% free
|
||||
}
|
||||
|
||||
return c1
|
||||
}
|
||||
|
||||
@@ -1314,19 +1371,9 @@ func (this *FileStorage) increaseHit(key string, hash string, reader Reader) {
|
||||
if rate <= 0 {
|
||||
rate = 1000
|
||||
}
|
||||
if this.lastHotSize == 0 {
|
||||
// 自动降低采样率来增加热点数据的缓存几率
|
||||
rate = rate / 10
|
||||
}
|
||||
if rands.Int(0, rate) == 0 {
|
||||
var memoryStorage = this.memoryStorage
|
||||
|
||||
var hitErr = this.list.IncreaseHit(hash)
|
||||
if hitErr != nil {
|
||||
// 此错误可以忽略
|
||||
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
|
||||
}
|
||||
|
||||
// 增加到热点
|
||||
// 这里不收录缓存尺寸过大的文件
|
||||
if memoryStorage != nil && reader.BodySize() > 0 && reader.BodySize() < 128*sizes.M {
|
||||
@@ -1342,6 +1389,15 @@ func (this *FileStorage) increaseHit(key string, hash string, reader Reader) {
|
||||
}
|
||||
}
|
||||
this.hotMapLocker.Unlock()
|
||||
|
||||
// 只有重复点击的才增加点击量
|
||||
if ok {
|
||||
var hitErr = this.list.IncreaseHit(hash)
|
||||
if hitErr != nil {
|
||||
// 此错误可以忽略
|
||||
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1362,7 +1418,11 @@ func (this *FileStorage) removeCacheFile(path string) error {
|
||||
if openFileCache != nil {
|
||||
openFileCache.Close(partialPath)
|
||||
}
|
||||
_ = os.Remove(partialPath)
|
||||
|
||||
_, statErr := os.Stat(partialPath)
|
||||
if statErr == nil {
|
||||
_ = os.Remove(partialPath)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -1460,6 +1520,7 @@ func (this *FileStorage) checkDiskSpace() {
|
||||
stat, err := fsutils.StatDevice(options.Dir)
|
||||
if err == nil {
|
||||
this.mainDiskIsFull = stat.FreeSize() < minFreeSize
|
||||
this.mainDiskTotalSize = stat.TotalSize()
|
||||
|
||||
// check capacity (only on main directory) when node capacity had not been set
|
||||
if !this.mainDiskIsFull {
|
||||
@@ -1549,6 +1610,15 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
allDirs = append(allDirs, subDir.Path)
|
||||
}
|
||||
|
||||
var countDirs = 0
|
||||
|
||||
// process progress
|
||||
var progressSock = gosock.NewTmpSock(teaconst.CacheGarbageSockName)
|
||||
_, sockErr := progressSock.SendTimeout(&gosock.Command{Code: "progress", Params: map[string]any{"progress": 0}}, 1*time.Second)
|
||||
var canReportProgress = sockErr == nil
|
||||
var lastProgress float64
|
||||
var countFound = 0
|
||||
|
||||
for _, subDir := range allDirs {
|
||||
var dir0 = subDir + "/p" + types.String(this.policy.Id)
|
||||
dir1Matches, err := filepath.Glob(dir0 + "/*")
|
||||
@@ -1572,6 +1642,20 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
continue
|
||||
}
|
||||
|
||||
countDirs++
|
||||
|
||||
// report progress
|
||||
if canReportProgress {
|
||||
var progress = float64(countDirs) / 65536
|
||||
if fmt.Sprintf("%.2f", lastProgress) != fmt.Sprintf("%.2f", progress) {
|
||||
lastProgress = progress
|
||||
_, _ = progressSock.SendTimeout(&gosock.Command{Code: "progress", Params: map[string]any{
|
||||
"progress": progress,
|
||||
"count": countFound,
|
||||
}}, 100*time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
fileMatches, err := filepath.Glob(dir2 + "/*.cache")
|
||||
if err != nil {
|
||||
// ignore error
|
||||
@@ -1604,6 +1688,7 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
}
|
||||
|
||||
if fileCallback != nil {
|
||||
countFound++
|
||||
err = fileCallback(file)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1614,6 +1699,14 @@ func (this *FileStorage) ScanGarbageCaches(fileCallback func(path string) error)
|
||||
}
|
||||
}
|
||||
|
||||
// 100% progress
|
||||
if canReportProgress && lastProgress != 1 {
|
||||
_, _ = progressSock.SendTimeout(&gosock.Command{Code: "progress", Params: map[string]any{
|
||||
"progress": 1,
|
||||
"count": countFound,
|
||||
}}, 100*time.Millisecond)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -592,9 +592,7 @@ func TestFileStorage_ScanGarbageCaches(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = storage.ScanGarbageCaches(func(path string) {
|
||||
t.Log(path)
|
||||
}, func(path string) error {
|
||||
err = storage.ScanGarbageCaches(func(path string) error {
|
||||
t.Log(path, PartialRangesFilePath(path))
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -9,11 +9,9 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/sizes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
"math"
|
||||
"runtime"
|
||||
"strconv"
|
||||
@@ -30,6 +28,11 @@ type MemoryItem struct {
|
||||
Status int
|
||||
IsDone bool
|
||||
ModifiedAt int64
|
||||
|
||||
IsPrepared bool
|
||||
WriteOffset int64
|
||||
|
||||
isReferring bool // if it is referring by other objects
|
||||
}
|
||||
|
||||
func (this *MemoryItem) IsExpired() bool {
|
||||
@@ -50,7 +53,7 @@ type MemoryStorage struct {
|
||||
|
||||
purgeTicker *utils.Ticker
|
||||
|
||||
totalSize int64
|
||||
usedSize int64
|
||||
writingKeyMap map[string]zero.Zero // key => bool
|
||||
|
||||
ignoreKeys *setutils.FixedSet
|
||||
@@ -62,7 +65,7 @@ func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage Stora
|
||||
|
||||
if parentStorage != nil {
|
||||
if queueSize <= 0 {
|
||||
queueSize = 2048 + int(policy.CapacityBytes()/sizes.G)*2048
|
||||
queueSize = utils.SystemMemoryGB() * 100_000
|
||||
}
|
||||
|
||||
dirtyChan = make(chan string, queueSize)
|
||||
@@ -85,10 +88,10 @@ func (this *MemoryStorage) Init() error {
|
||||
_ = this.list.Init()
|
||||
|
||||
this.list.OnAdd(func(item *Item) {
|
||||
atomic.AddInt64(&this.totalSize, item.TotalSize())
|
||||
atomic.AddInt64(&this.usedSize, item.TotalSize())
|
||||
})
|
||||
this.list.OnRemove(func(item *Item) {
|
||||
atomic.AddInt64(&this.totalSize, -item.TotalSize())
|
||||
atomic.AddInt64(&this.usedSize, -item.TotalSize())
|
||||
})
|
||||
|
||||
this.initPurgeTicker()
|
||||
@@ -121,7 +124,12 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool)
|
||||
|
||||
// read from valuesMap
|
||||
this.locker.RLock()
|
||||
item := this.valuesMap[hash]
|
||||
var item = this.valuesMap[hash]
|
||||
|
||||
if item != nil {
|
||||
item.isReferring = true
|
||||
}
|
||||
|
||||
if item == nil || !item.IsDone {
|
||||
this.locker.RUnlock()
|
||||
return nil, ErrNotFound
|
||||
@@ -168,7 +176,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
|
||||
if isDirty &&
|
||||
this.parentStorage != nil &&
|
||||
this.dirtyQueueSize > 0 &&
|
||||
len(this.dirtyChan) == this.dirtyQueueSize { // 缓存时间过长
|
||||
len(this.dirtyChan) >= this.dirtyQueueSize-int(fsutils.DiskMaxWrites) /** delta **/ { // 缓存时间过长
|
||||
return nil, ErrWritingQueueFull
|
||||
}
|
||||
|
||||
@@ -204,13 +212,13 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否超出最大值
|
||||
capacityBytes := this.memoryCapacityBytes()
|
||||
// 检查是否超出容量最大值
|
||||
var capacityBytes = this.memoryCapacityBytes()
|
||||
if bodySize < 0 {
|
||||
bodySize = 0
|
||||
}
|
||||
if capacityBytes > 0 && capacityBytes <= this.totalSize+bodySize {
|
||||
return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
|
||||
if capacityBytes > 0 && capacityBytes <= atomic.LoadInt64(&this.usedSize)+bodySize {
|
||||
return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.usedSize, 10) + " bytes")
|
||||
}
|
||||
|
||||
// 先删除
|
||||
@@ -220,7 +228,7 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, h
|
||||
}
|
||||
|
||||
isWriting = true
|
||||
return NewMemoryWriter(this, key, expiresAt, status, isDirty, maxSize, func() {
|
||||
return NewMemoryWriter(this, key, expiresAt, status, isDirty, bodySize, maxSize, func(valueItem *MemoryItem) {
|
||||
this.locker.Lock()
|
||||
delete(this.writingKeyMap, key)
|
||||
this.locker.Unlock()
|
||||
@@ -252,7 +260,7 @@ func (this *MemoryStorage) CleanAll() error {
|
||||
this.locker.Lock()
|
||||
this.valuesMap = map[uint64]*MemoryItem{}
|
||||
_ = this.list.Reset()
|
||||
atomic.StoreInt64(&this.totalSize, 0)
|
||||
atomic.StoreInt64(&this.usedSize, 0)
|
||||
this.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -363,6 +371,11 @@ func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePol
|
||||
|
||||
// AddToList 将缓存添加到列表
|
||||
func (this *MemoryStorage) AddToList(item *Item) {
|
||||
// skip added item
|
||||
if item.MetaSize > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/
|
||||
var hash = types.String(this.hash(item.Key))
|
||||
|
||||
@@ -380,7 +393,7 @@ func (this *MemoryStorage) TotalDiskSize() int64 {
|
||||
|
||||
// TotalMemorySize 内存尺寸
|
||||
func (this *MemoryStorage) TotalMemorySize() int64 {
|
||||
return atomic.LoadInt64(&this.totalSize)
|
||||
return atomic.LoadInt64(&this.usedSize)
|
||||
}
|
||||
|
||||
// IgnoreKey 忽略某个Key,即不缓存某个Key
|
||||
@@ -393,6 +406,11 @@ func (this *MemoryStorage) CanSendfile() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// HasFreeSpaceForHotItems 是否有足够的空间提供给热门内容
|
||||
func (this *MemoryStorage) HasFreeSpaceForHotItems() bool {
|
||||
return atomic.LoadInt64(&this.usedSize) < this.memoryCapacityBytes()*3/4
|
||||
}
|
||||
|
||||
// 计算Key Hash
|
||||
func (this *MemoryStorage) hash(key string) uint64 {
|
||||
return xxhash.Sum64String(key)
|
||||
@@ -400,22 +418,6 @@ func (this *MemoryStorage) hash(key string) uint64 {
|
||||
|
||||
// 清理任务
|
||||
func (this *MemoryStorage) purgeLoop() {
|
||||
// 计算是否应该开启LFU清理
|
||||
var capacityBytes = this.policy.CapacityBytes()
|
||||
var startLFU = false
|
||||
var usedPercent = float32(this.TotalMemorySize()*100) / float32(capacityBytes)
|
||||
var lfuFreePercent = this.policy.MemoryLFUFreePercent
|
||||
if lfuFreePercent <= 0 {
|
||||
lfuFreePercent = 5
|
||||
}
|
||||
if capacityBytes > 0 {
|
||||
if lfuFreePercent < 100 {
|
||||
if usedPercent >= 100-lfuFreePercent {
|
||||
startLFU = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理过期
|
||||
var purgeCount = this.policy.MemoryAutoPurgeCount
|
||||
if purgeCount <= 0 {
|
||||
@@ -432,6 +434,23 @@ func (this *MemoryStorage) purgeLoop() {
|
||||
})
|
||||
|
||||
// LFU
|
||||
// 计算是否应该开启LFU清理
|
||||
var capacityBytes = this.policy.CapacityBytes()
|
||||
var startLFU = false
|
||||
|
||||
var usedPercent = float32(this.TotalMemorySize()*100) / float32(capacityBytes)
|
||||
var lfuFreePercent = this.policy.MemoryLFUFreePercent
|
||||
if lfuFreePercent <= 0 {
|
||||
lfuFreePercent = 5
|
||||
}
|
||||
if capacityBytes > 0 {
|
||||
if lfuFreePercent < 100 {
|
||||
if usedPercent >= 100-lfuFreePercent {
|
||||
startLFU = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if startLFU {
|
||||
var total, _ = this.list.Count()
|
||||
if total > 0 {
|
||||
@@ -464,33 +483,20 @@ func (this *MemoryStorage) purgeLoop() {
|
||||
// 开始Flush任务
|
||||
func (this *MemoryStorage) startFlush() {
|
||||
var statCount = 0
|
||||
var writeDelayMS float64 = 0
|
||||
|
||||
for key := range this.dirtyChan {
|
||||
statCount++
|
||||
|
||||
if statCount == 100 {
|
||||
statCount = 0
|
||||
|
||||
// delay some time to reduce load if needed
|
||||
if !fsutils.DiskIsFast() {
|
||||
loadStat, err := load.Avg()
|
||||
if err == nil && loadStat != nil {
|
||||
if loadStat.Load1 > 10 {
|
||||
writeDelayMS = 100
|
||||
} else if loadStat.Load1 > 5 {
|
||||
writeDelayMS = 50
|
||||
} else {
|
||||
writeDelayMS = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.flushItem(key)
|
||||
|
||||
if writeDelayMS > 0 {
|
||||
time.Sleep(time.Duration(writeDelayMS) * time.Millisecond)
|
||||
if fsutils.IsInExtremelyHighLoad {
|
||||
time.Sleep(1 * time.Second)
|
||||
} else if fsutils.IsInHighLoad {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -506,9 +512,20 @@ func (this *MemoryStorage) flushItem(key string) {
|
||||
item, ok := this.valuesMap[hash]
|
||||
this.locker.RUnlock()
|
||||
|
||||
// 从内存中移除,并确保无论如何都会执行
|
||||
defer func() {
|
||||
_ = this.Delete(key)
|
||||
|
||||
// 重用内存,前提是确保内存不再被引用
|
||||
if enableFragmentPool && ok && item.IsDone && !item.isReferring && len(item.BodyValue) > 0 {
|
||||
SharedFragmentMemoryPool.Put(item.BodyValue)
|
||||
}
|
||||
}()
|
||||
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if !item.IsDone {
|
||||
remotelogs.Error("CACHE", "flush items failed: open writer failed: item has not been done")
|
||||
return
|
||||
@@ -517,6 +534,16 @@ func (this *MemoryStorage) flushItem(key string) {
|
||||
return
|
||||
}
|
||||
|
||||
// 检查是否在列表中,防止未加入列表时就开始flush
|
||||
isInList, err := this.list.Exist(types.String(hash))
|
||||
if err != nil {
|
||||
remotelogs.Error("CACHE", "flush items failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !isInList {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
writer, err := this.parentStorage.OpenFlushWriter(key, item.ExpiresAt, item.Status, len(item.HeaderValue), int64(len(item.BodyValue)))
|
||||
if err != nil {
|
||||
if !CanIgnoreErr(err) {
|
||||
@@ -554,26 +581,37 @@ func (this *MemoryStorage) flushItem(key string) {
|
||||
HeaderSize: writer.HeaderSize(),
|
||||
BodySize: writer.BodySize(),
|
||||
})
|
||||
|
||||
// 从内存中移除
|
||||
_ = this.Delete(key)
|
||||
}
|
||||
|
||||
func (this *MemoryStorage) memoryCapacityBytes() int64 {
|
||||
var maxSystemBytes = SharedManager.MaxSystemMemoryBytesPerStorage()
|
||||
if this.policy == nil {
|
||||
return 0
|
||||
}
|
||||
c1 := int64(0)
|
||||
if this.policy.Capacity != nil {
|
||||
c1 = this.policy.Capacity.Bytes()
|
||||
return maxSystemBytes
|
||||
}
|
||||
|
||||
if SharedManager.MaxMemoryCapacity != nil {
|
||||
c2 := SharedManager.MaxMemoryCapacity.Bytes()
|
||||
if c2 > 0 {
|
||||
return c2
|
||||
var capacityBytes = SharedManager.MaxMemoryCapacity.Bytes()
|
||||
if capacityBytes > 0 {
|
||||
if capacityBytes > maxSystemBytes {
|
||||
return maxSystemBytes
|
||||
}
|
||||
|
||||
return capacityBytes
|
||||
}
|
||||
}
|
||||
return c1
|
||||
|
||||
var capacity = this.policy.Capacity // copy
|
||||
if capacity != nil {
|
||||
var capacityBytes = capacity.Bytes()
|
||||
if capacityBytes > 0 {
|
||||
if capacityBytes > maxSystemBytes {
|
||||
return maxSystemBytes
|
||||
}
|
||||
return capacityBytes
|
||||
}
|
||||
}
|
||||
|
||||
return maxSystemBytes
|
||||
}
|
||||
|
||||
func (this *MemoryStorage) deleteWithoutLocker(key string) error {
|
||||
|
||||
@@ -2,9 +2,9 @@ package caches
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/cespare/xxhash"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MemoryWriter struct {
|
||||
@@ -16,29 +16,51 @@ type MemoryWriter struct {
|
||||
bodySize int64
|
||||
status int
|
||||
isDirty bool
|
||||
maxSize int64
|
||||
|
||||
expectedBodySize int64
|
||||
maxSize int64
|
||||
|
||||
hash uint64
|
||||
item *MemoryItem
|
||||
endFunc func()
|
||||
endFunc func(valueItem *MemoryItem)
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, maxSize int64, endFunc func()) *MemoryWriter {
|
||||
w := &MemoryWriter{
|
||||
storage: memoryStorage,
|
||||
key: key,
|
||||
expiredAt: expiredAt,
|
||||
item: &MemoryItem{
|
||||
ExpiresAt: expiredAt,
|
||||
ModifiedAt: time.Now().Unix(),
|
||||
Status: status,
|
||||
},
|
||||
status: status,
|
||||
isDirty: isDirty,
|
||||
maxSize: maxSize,
|
||||
endFunc: endFunc,
|
||||
func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, expectedBodySize int64, maxSize int64, endFunc func(valueItem *MemoryItem)) *MemoryWriter {
|
||||
var valueItem = &MemoryItem{
|
||||
ExpiresAt: expiredAt,
|
||||
ModifiedAt: fasttime.Now().Unix(),
|
||||
Status: status,
|
||||
}
|
||||
if enableFragmentPool &&
|
||||
expectedBodySize > 0 &&
|
||||
expectedBodySize <= maxMemoryFragmentPoolItemSize {
|
||||
bodyBytes, ok := SharedFragmentMemoryPool.Get(expectedBodySize) // try to reuse memory
|
||||
if ok {
|
||||
valueItem.BodyValue = bodyBytes
|
||||
valueItem.IsPrepared = true
|
||||
} else {
|
||||
if expectedBodySize <= (16 << 20) {
|
||||
var allocSize = (expectedBodySize/16384 + 1) * 16384
|
||||
valueItem.BodyValue = make([]byte, allocSize)[:expectedBodySize]
|
||||
valueItem.IsPrepared = true
|
||||
|
||||
SharedFragmentMemoryPool.IncreaseNew()
|
||||
}
|
||||
}
|
||||
}
|
||||
var w = &MemoryWriter{
|
||||
storage: memoryStorage,
|
||||
key: key,
|
||||
expiredAt: expiredAt,
|
||||
item: valueItem,
|
||||
status: status,
|
||||
isDirty: isDirty,
|
||||
expectedBodySize: expectedBodySize,
|
||||
maxSize: maxSize,
|
||||
endFunc: endFunc,
|
||||
}
|
||||
|
||||
w.hash = w.calculateHash(key)
|
||||
|
||||
return w
|
||||
@@ -53,17 +75,32 @@ func (this *MemoryWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
|
||||
// Write 写入数据
|
||||
func (this *MemoryWriter) Write(data []byte) (n int, err error) {
|
||||
this.bodySize += int64(len(data))
|
||||
this.item.BodyValue = append(this.item.BodyValue, data...)
|
||||
var l = len(data)
|
||||
if l == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if this.item.IsPrepared {
|
||||
if this.item.WriteOffset+int64(l) > this.expectedBodySize {
|
||||
err = ErrWritingUnavailable
|
||||
return
|
||||
}
|
||||
copy(this.item.BodyValue[this.item.WriteOffset:], data)
|
||||
this.item.WriteOffset += int64(l)
|
||||
} else {
|
||||
this.item.BodyValue = append(this.item.BodyValue, data...)
|
||||
}
|
||||
|
||||
this.bodySize += int64(l)
|
||||
|
||||
// 检查尺寸
|
||||
if this.maxSize > 0 && this.bodySize > this.maxSize {
|
||||
err = ErrEntityTooLarge
|
||||
this.storage.IgnoreKey(this.key, this.maxSize)
|
||||
return len(data), err
|
||||
return l, err
|
||||
}
|
||||
|
||||
return len(data), nil
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// WriteAt 在指定位置写入数据
|
||||
@@ -87,7 +124,8 @@ func (this *MemoryWriter) BodySize() int64 {
|
||||
func (this *MemoryWriter) Close() error {
|
||||
// 需要在Locker之外
|
||||
defer this.once.Do(func() {
|
||||
this.endFunc()
|
||||
this.endFunc(this.item)
|
||||
this.item = nil // free memory
|
||||
})
|
||||
|
||||
if this.item == nil {
|
||||
@@ -96,30 +134,49 @@ func (this *MemoryWriter) Close() error {
|
||||
|
||||
this.storage.locker.Lock()
|
||||
this.item.IsDone = true
|
||||
this.storage.valuesMap[this.hash] = this.item
|
||||
var err error
|
||||
if this.isDirty {
|
||||
if this.storage.parentStorage != nil {
|
||||
this.storage.valuesMap[this.hash] = this.item
|
||||
|
||||
select {
|
||||
case this.storage.dirtyChan <- this.key:
|
||||
default:
|
||||
// remove from values map
|
||||
delete(this.storage.valuesMap, this.hash)
|
||||
|
||||
err = ErrWritingQueueFull
|
||||
}
|
||||
} else {
|
||||
this.storage.valuesMap[this.hash] = this.item
|
||||
}
|
||||
} else {
|
||||
this.storage.valuesMap[this.hash] = this.item
|
||||
}
|
||||
|
||||
this.storage.locker.Unlock()
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
// Discard 丢弃
|
||||
func (this *MemoryWriter) Discard() error {
|
||||
// 需要在Locker之外
|
||||
defer this.once.Do(func() {
|
||||
this.endFunc()
|
||||
this.endFunc(this.item)
|
||||
this.item = nil // free memory
|
||||
})
|
||||
|
||||
this.storage.locker.Lock()
|
||||
delete(this.storage.valuesMap, this.hash)
|
||||
|
||||
if enableFragmentPool &&
|
||||
this.item != nil &&
|
||||
!this.item.isReferring &&
|
||||
cap(this.item.BodyValue) >= minMemoryFragmentPoolItemSize {
|
||||
SharedFragmentMemoryPool.Put(this.item.BodyValue)
|
||||
}
|
||||
|
||||
this.storage.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build !plus || !linux
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build !plus || !linux
|
||||
|
||||
package compressions
|
||||
|
||||
@@ -27,7 +28,7 @@ func newBrotliWriter(writer io.Writer, level int) (*BrotliWriter, error) {
|
||||
return &BrotliWriter{
|
||||
writer: brotli.NewWriterOptions(writer, brotli.WriterOptions{
|
||||
Quality: level,
|
||||
LGWin: 13, // TODO 在全局设置里可以设置此值
|
||||
LGWin: 14, // TODO 在全局设置里可以设置此值
|
||||
}),
|
||||
level: level,
|
||||
}, nil
|
||||
|
||||
@@ -19,6 +19,10 @@ func NewZSTDWriter(writer io.Writer, level int) (Writer, error) {
|
||||
}
|
||||
|
||||
func newZSTDWriter(writer io.Writer, level int) (Writer, error) {
|
||||
if level < 0 {
|
||||
level = 0
|
||||
}
|
||||
|
||||
var zstdLevel = zstd.EncoderLevelFromZstd(level)
|
||||
|
||||
zstdWriter, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstdLevel))
|
||||
|
||||
@@ -9,6 +9,24 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewZSTDWriter_Level0(t *testing.T) {
|
||||
var buf = &bytes.Buffer{}
|
||||
writer, err := compressions.NewZSTDWriter(buf, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var originData = []byte(strings.Repeat("Hello", 1024))
|
||||
_, err = writer.Write(originData)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("origin data:", len(originData), "result:", buf.Len())
|
||||
}
|
||||
|
||||
func TestNewZSTDWriter(t *testing.T) {
|
||||
var buf = &bytes.Buffer{}
|
||||
writer, err := compressions.NewZSTDWriter(buf, 10)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.2.9"
|
||||
Version = "1.3.2"
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
@@ -14,5 +14,6 @@ const (
|
||||
// SystemdServiceName systemd
|
||||
SystemdServiceName = "edge-node"
|
||||
|
||||
AccessLogSockName = "edge-node.accesslog.sock"
|
||||
AccessLogSockName = "edge-node.accesslog"
|
||||
CacheGarbageSockName = "edge-node.cache.garbage"
|
||||
)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/expires"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
@@ -150,7 +149,6 @@ func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found b
|
||||
|
||||
func (this *IPList) SetDeleted() {
|
||||
this.isDeleted = true
|
||||
logs.Println("set deleted:", this.isDeleted) // TODO
|
||||
}
|
||||
|
||||
func (this *IPList) addItem(item *IPItem, sortable bool) {
|
||||
|
||||
@@ -60,7 +60,7 @@ func (this *IPListDB) init() error {
|
||||
|
||||
var path = this.dir + "/ip_list.db"
|
||||
|
||||
db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
|
||||
db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func (this *Task) Init() error {
|
||||
|
||||
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
|
||||
|
||||
db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
|
||||
db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_locking_mode=EXCLUSIVE")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -90,13 +90,13 @@ func (this *APIStream) loop() error {
|
||||
break
|
||||
}
|
||||
|
||||
message, err := nodeStream.Recv()
|
||||
if err != nil {
|
||||
message, streamErr := nodeStream.Recv()
|
||||
if streamErr != nil {
|
||||
if this.isQuiting {
|
||||
remotelogs.Println("API_STREAM", "quit")
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
return streamErr
|
||||
}
|
||||
|
||||
// 处理消息
|
||||
|
||||
@@ -24,8 +24,6 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var synFloodCounter = counters.NewCounter().WithGC()
|
||||
|
||||
// ClientConn 客户端连接
|
||||
type ClientConn struct {
|
||||
BaseClientConn
|
||||
@@ -292,13 +290,13 @@ func (this *ClientConn) LastErr() error {
|
||||
}
|
||||
|
||||
func (this *ClientConn) resetSYNFlood() {
|
||||
synFloodCounter.ResetKey("SYN_FLOOD:" + this.RawIP())
|
||||
counters.SharedCounter.ResetKey("SYN_FLOOD:" + this.RawIP())
|
||||
}
|
||||
|
||||
func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloodConfig) {
|
||||
var ip = this.RawIP()
|
||||
if len(ip) > 0 && !iplibrary.IsInWhiteList(ip) && (!synFloodConfig.IgnoreLocal || !utils.IsLocalIP(ip)) {
|
||||
var result = synFloodCounter.IncreaseKey("SYN_FLOOD:"+ip, 60)
|
||||
var result = counters.SharedCounter.IncreaseKey("SYN_FLOOD:"+ip, 60)
|
||||
var minAttempts = synFloodConfig.MinAttempts
|
||||
if minAttempts < 5 {
|
||||
minAttempts = 5
|
||||
@@ -307,7 +305,7 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
|
||||
// 非TLS,设置为两倍,防止误封
|
||||
minAttempts = 2 * minAttempts
|
||||
}
|
||||
if result >= types.Uint64(minAttempts) {
|
||||
if result >= types.Uint32(minAttempts) {
|
||||
var timeout = synFloodConfig.TimeoutSeconds
|
||||
if timeout <= 0 {
|
||||
timeout = 600
|
||||
|
||||
@@ -85,6 +85,8 @@ type HTTPRequest struct {
|
||||
isAttack bool // 是否是攻击请求
|
||||
requestBodyData []byte // 读取的Body内容
|
||||
|
||||
isWebsocketResponse bool // 是否为Websocket响应(非请求)
|
||||
|
||||
// WAF相关
|
||||
firewallPolicyId int64
|
||||
firewallRuleGroupId int64
|
||||
@@ -204,6 +206,14 @@ func (this *HTTPRequest) Do() {
|
||||
return
|
||||
}
|
||||
|
||||
// WAF
|
||||
if this.web.FirewallRef != nil && this.web.FirewallRef.IsOn {
|
||||
if this.doWAFRequest() {
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// UAM
|
||||
if !this.isHealthCheck {
|
||||
if this.web.UAM != nil {
|
||||
@@ -234,14 +244,6 @@ func (this *HTTPRequest) Do() {
|
||||
}
|
||||
}
|
||||
|
||||
// WAF
|
||||
if this.web.FirewallRef != nil && this.web.FirewallRef.IsOn {
|
||||
if this.doWAFRequest() {
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 防盗链
|
||||
if !this.isSubRequest && this.web.Referers != nil && this.web.Referers.IsOn {
|
||||
if this.doCheckReferers() {
|
||||
@@ -410,6 +412,8 @@ func (this *HTTPRequest) doEnd() {
|
||||
var countAttacks int64 = 0
|
||||
var attackBytes int64 = 0
|
||||
|
||||
var countWebsocketConnections int64 = 0
|
||||
|
||||
if this.isCached {
|
||||
countCached = 1
|
||||
cachedBytes = totalBytes
|
||||
@@ -421,8 +425,11 @@ func (this *HTTPRequest) doEnd() {
|
||||
attackBytes = totalBytes
|
||||
}
|
||||
}
|
||||
if this.isWebsocketResponse {
|
||||
countWebsocketConnections = 1
|
||||
}
|
||||
|
||||
stats.SharedTrafficStatManager.Add(this.ReqServer.UserId, this.ReqServer.Id, this.ReqHost, totalBytes, cachedBytes, 1, countCached, countAttacks, attackBytes, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(this.ReqServer.UserId, this.ReqServer.Id, this.ReqHost, totalBytes, cachedBytes, 1, countCached, countAttacks, attackBytes, countWebsocketConnections, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
|
||||
|
||||
// 指标
|
||||
if metrics.SharedManager.HasHTTPMetrics() {
|
||||
|
||||
@@ -3,6 +3,7 @@ package nodes
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
@@ -130,7 +131,22 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
var tags = []string{}
|
||||
|
||||
// 检查是否有缓存
|
||||
var key = this.Format(this.cacheRef.Key)
|
||||
var key string
|
||||
if this.web.Cache.Key != nil && this.web.Cache.Key.IsOn && len(this.web.Cache.Key.Host) > 0 {
|
||||
key = configutils.ParseVariables(this.cacheRef.Key, func(varName string) (value string) {
|
||||
switch varName {
|
||||
case "scheme":
|
||||
return this.web.Cache.Key.Scheme
|
||||
case "host":
|
||||
return this.web.Cache.Key.Host
|
||||
default:
|
||||
return this.Format("${" + varName + "}")
|
||||
}
|
||||
})
|
||||
} else {
|
||||
key = this.Format(this.cacheRef.Key)
|
||||
}
|
||||
|
||||
if len(key) == 0 {
|
||||
this.cacheRef = nil
|
||||
cacheBypassDescription = "BYPASS, empty key"
|
||||
@@ -274,7 +290,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err == caches.ErrNotFound {
|
||||
if errors.Is(err, caches.ErrNotFound) {
|
||||
// 移除请求中的 If-None-Match 和 If-Modified-Since,防止源站返回304而无法缓存
|
||||
if this.reverseProxy != nil {
|
||||
this.RawReq.Header.Del("If-None-Match")
|
||||
this.RawReq.Header.Del("If-Modified-Since")
|
||||
}
|
||||
|
||||
// cache相关变量
|
||||
this.varMapping["cache.status"] = "MISS"
|
||||
|
||||
@@ -365,24 +387,24 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
}
|
||||
|
||||
// ETag
|
||||
// 这里强制设置ETag,如果先前源站设置了ETag,将会被覆盖,避免因为源站的ETag导致源站返回304 Not Modified
|
||||
var respHeader = this.writer.Header()
|
||||
var eTag = ""
|
||||
var eTag = respHeader.Get("ETag")
|
||||
var lastModifiedAt = reader.LastModified()
|
||||
if lastModifiedAt > 0 {
|
||||
if len(tags) > 0 {
|
||||
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
|
||||
} else {
|
||||
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
|
||||
}
|
||||
respHeader.Del("Etag")
|
||||
if !isPartialCache {
|
||||
respHeader["ETag"] = []string{eTag}
|
||||
if len(eTag) == 0 {
|
||||
if lastModifiedAt > 0 {
|
||||
if len(tags) > 0 {
|
||||
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
|
||||
} else {
|
||||
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
|
||||
}
|
||||
respHeader.Del("Etag")
|
||||
if !isPartialCache {
|
||||
respHeader["ETag"] = []string{eTag}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 支持 Last-Modified
|
||||
// 这里强制设置Last-Modified,如果先前源站设置了Last-Modified,将会被覆盖,避免因为源站的Last-Modified导致源站返回304 Not Modified
|
||||
var modifiedTime = ""
|
||||
if lastModifiedAt > 0 {
|
||||
modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
|
||||
@@ -490,7 +512,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
if err != nil {
|
||||
this.varMapping["cache.status"] = "MISS"
|
||||
|
||||
if err == caches.ErrInvalidRange {
|
||||
if errors.Is(err, caches.ErrInvalidRange) {
|
||||
this.ProcessResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
|
||||
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return true
|
||||
|
||||
@@ -25,6 +25,13 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(u.ExceptDomains) > 0 && configutils.MatchDomains(u.ExceptDomains, this.ReqHost) {
|
||||
continue
|
||||
}
|
||||
if len(u.OnlyDomains) > 0 && !configutils.MatchDomains(u.OnlyDomains, this.ReqHost) {
|
||||
continue
|
||||
}
|
||||
|
||||
var status = u.Status
|
||||
if status <= 0 {
|
||||
if searchEngineRegex.MatchString(this.RawReq.UserAgent()) {
|
||||
@@ -139,11 +146,6 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// 如果跳转前后域名一致,则终止
|
||||
if u.DomainAfter == reqHost {
|
||||
return false
|
||||
}
|
||||
|
||||
var scheme = u.DomainAfterScheme
|
||||
if len(scheme) == 0 {
|
||||
scheme = this.requestScheme()
|
||||
@@ -155,6 +157,11 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
|
||||
return false
|
||||
}
|
||||
|
||||
// 如果跳转前后域名一致,则终止
|
||||
if u.DomainAfter == reqHost {
|
||||
return false
|
||||
}
|
||||
|
||||
this.ProcessResponseHeaders(this.writer.Header(), status)
|
||||
|
||||
// 参数
|
||||
|
||||
@@ -20,6 +20,6 @@ func (this *HTTPRequest) checkLnRequest() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
|
||||
func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64, urlHash uint64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
|
||||
return nil, 0, false
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ func (this *HTTPRequest) doMismatch() {
|
||||
// 要考虑到服务在切换集群时,域名未生效状态时,用户访问的仍然是老集群中的节点,就会产生找不到域名的情况
|
||||
if len(remoteIP) > 0 {
|
||||
const maxAttempts = 100
|
||||
if ttlcache.SharedCache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
|
||||
if ttlcache.SharedInt64Cache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
|
||||
// 在加入之前再次检查黑名单
|
||||
if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
|
||||
waf.SharedIPBlackList.Add(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+3600)
|
||||
|
||||
@@ -2,7 +2,6 @@ package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
@@ -46,9 +45,13 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, status int) (shouldStop bool) {
|
||||
var url = this.URL()
|
||||
for _, page := range pages {
|
||||
if !page.MatchURL(url) {
|
||||
continue
|
||||
}
|
||||
if page.Match(status) {
|
||||
if len(page.BodyType) == 0 || page.BodyType == shared.BodyTypeURL {
|
||||
if len(page.BodyType) == 0 || page.BodyType == serverconfigs.HTTPPageBodyTypeURL {
|
||||
if urlSchemeRegexp.MatchString(page.URL) {
|
||||
var newStatus = page.NewStatus
|
||||
if newStatus <= 0 {
|
||||
@@ -115,7 +118,7 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
|
||||
}
|
||||
|
||||
return true
|
||||
} else if page.BodyType == shared.BodyTypeHTML {
|
||||
} else if page.BodyType == serverconfigs.HTTPPageBodyTypeHTML {
|
||||
// 这里需要实现设置Status,因为在Format()中可以获取${status}等变量
|
||||
if page.NewStatus > 0 {
|
||||
this.writer.statusCode = page.NewStatus
|
||||
@@ -147,6 +150,18 @@ func (this *HTTPRequest) doPageLookup(pages []*serverconfigs.HTTPPageConfig, sta
|
||||
this.writer.SetOk()
|
||||
}
|
||||
return true
|
||||
} else if page.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
|
||||
var newURL = page.URL
|
||||
if len(newURL) == 0 {
|
||||
newURL = "/"
|
||||
}
|
||||
if page.NewStatus > 0 && httpStatusIsRedirect(page.NewStatus) {
|
||||
httpRedirect(this.writer, this.RawReq, newURL, page.NewStatus)
|
||||
} else {
|
||||
httpRedirect(this.writer, this.RawReq, newURL, http.StatusTemporaryRedirect)
|
||||
}
|
||||
this.writer.SetOk()
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,9 +27,10 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
|
||||
var failedOriginIds []int64
|
||||
var failedLnNodeIds []int64
|
||||
var failStatusCode int
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1)
|
||||
originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1, &failStatusCode)
|
||||
if !shouldRetry {
|
||||
break
|
||||
}
|
||||
@@ -43,7 +44,7 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
}
|
||||
|
||||
// 请求源站
|
||||
func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool) (originId int64, lnNodeId int64, shouldRetry bool) {
|
||||
func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool, failStatusCode *int) (originId int64, lnNodeId int64, shouldRetry bool) {
|
||||
// 对URL的处理
|
||||
var stripPrefix = this.reverseProxy.StripPrefix
|
||||
var requestURI = this.reverseProxy.RequestURI
|
||||
@@ -91,6 +92,10 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
}
|
||||
if origin == nil {
|
||||
origin = this.reverseProxy.NextOrigin(requestCall)
|
||||
if origin != nil && origin.Id > 0 && (*failStatusCode >= 403 && *failStatusCode <= 404) && lists.ContainsInt64(failedOriginIds, origin.Id) {
|
||||
this.writeCode(*failStatusCode, "", "")
|
||||
return
|
||||
}
|
||||
}
|
||||
requestCall.CallResponseCallbacks(this.writer)
|
||||
if origin == nil {
|
||||
@@ -335,7 +340,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
shouldRetry = true
|
||||
this.uri = oldURI // 恢复备份
|
||||
|
||||
if httpErr.Err != io.EOF {
|
||||
if httpErr.Err != io.EOF && !errors.Is(httpErr.Err, http.ErrBodyReadAfterClose) {
|
||||
remotelogs.WarnServer("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+requestErr.Error())
|
||||
}
|
||||
|
||||
@@ -349,7 +354,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
} else {
|
||||
this.write50x(requestErr, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
|
||||
}
|
||||
if httpErr.Err != io.EOF {
|
||||
if httpErr.Err != io.EOF && !errors.Is(httpErr.Err, http.ErrBodyReadAfterClose) {
|
||||
remotelogs.WarnServer("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+requestErr.Error())
|
||||
}
|
||||
} else {
|
||||
@@ -376,11 +381,11 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
return
|
||||
}
|
||||
|
||||
// 50x
|
||||
// 40x && 50x
|
||||
*failStatusCode = resp.StatusCode
|
||||
if resp != nil &&
|
||||
resp.StatusCode >= 500 &&
|
||||
resp.StatusCode < 510 &&
|
||||
this.reverseProxy.Retry50X &&
|
||||
((resp.StatusCode >= 500 && resp.StatusCode < 510 && this.reverseProxy.Retry50X) ||
|
||||
(resp.StatusCode >= 403 && resp.StatusCode <= 404 && this.reverseProxy.Retry40X)) &&
|
||||
(originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) &&
|
||||
!isLastRetry {
|
||||
if resp.Body != nil {
|
||||
@@ -429,7 +434,7 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
|
||||
// Page optimization
|
||||
if this.web.Optimization != nil && resp.Body != nil && this.cacheRef != nil /** must under cache **/ {
|
||||
err := this.web.Optimization.FilterResponse(resp)
|
||||
err := this.web.Optimization.FilterResponse(this.URL(), resp)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusBadGateway, "Page Optimization: Fail to read content from origin", "内容优化:从源站读取内容失败", false)
|
||||
return
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
@@ -18,7 +18,7 @@ func (this *HTTPRequest) doShutdown() {
|
||||
return
|
||||
}
|
||||
|
||||
if len(shutdown.BodyType) == 0 || shutdown.BodyType == shared.BodyTypeURL {
|
||||
if len(shutdown.BodyType) == 0 || shutdown.BodyType == serverconfigs.HTTPPageBodyTypeURL {
|
||||
// URL
|
||||
if urlSchemeRegexp.MatchString(shutdown.URL) {
|
||||
this.doURL(http.MethodGet, shutdown.URL, "", shutdown.Status, true)
|
||||
@@ -80,7 +80,7 @@ func (this *HTTPRequest) doShutdown() {
|
||||
} else {
|
||||
this.writer.SetOk()
|
||||
}
|
||||
} else if shutdown.BodyType == shared.BodyTypeHTML {
|
||||
} else if shutdown.BodyType == serverconfigs.HTTPPageBodyTypeHTML {
|
||||
// 自定义响应Headers
|
||||
if shutdown.Status > 0 {
|
||||
this.ProcessResponseHeaders(this.writer.Header(), shutdown.Status)
|
||||
@@ -98,5 +98,17 @@ func (this *HTTPRequest) doShutdown() {
|
||||
} else {
|
||||
this.writer.SetOk()
|
||||
}
|
||||
} else if shutdown.BodyType == serverconfigs.HTTPPageBodyTypeRedirectURL {
|
||||
var newURL = shutdown.URL
|
||||
if len(newURL) == 0 {
|
||||
newURL = "/"
|
||||
}
|
||||
|
||||
if shutdown.Status > 0 && httpStatusIsRedirect(shutdown.Status) {
|
||||
httpRedirect(this.writer, this.RawReq, newURL, shutdown.Status)
|
||||
} else {
|
||||
httpRedirect(this.writer, this.RawReq, newURL, http.StatusTemporaryRedirect)
|
||||
}
|
||||
this.writer.SetOk()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,6 +96,8 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
return
|
||||
}
|
||||
|
||||
var isDefendMode = firewallPolicy.Mode == firewallconfigs.FirewallModeDefend
|
||||
|
||||
// 检查IP白名单
|
||||
var remoteAddrs []string
|
||||
if len(this.remoteAddr) > 0 {
|
||||
@@ -122,7 +124,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
}
|
||||
|
||||
// 检查IP黑名单
|
||||
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
|
||||
if isDefendMode {
|
||||
for _, ref := range inbound.AllDenyListRefs() {
|
||||
if ref.IsOn && ref.ListId > 0 {
|
||||
list := iplibrary.SharedIPListManager.FindList(ref.ListId)
|
||||
@@ -161,19 +163,20 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
}
|
||||
|
||||
// 检查地区封禁
|
||||
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
|
||||
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
|
||||
var regionConfig = firewallPolicy.Inbound.Region
|
||||
if regionConfig.IsNotEmpty() {
|
||||
for _, remoteAddr := range remoteAddrs {
|
||||
var result = iplib.LookupIP(remoteAddr)
|
||||
if result != nil && result.IsOk() {
|
||||
var currentURL = this.URL()
|
||||
if regionConfig.MatchCountryURL(currentURL) {
|
||||
// 检查国家/地区级别封禁
|
||||
if !regionConfig.IsAllowedCountry(result.CountryId(), result.ProvinceId()) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
|
||||
var regionConfig = firewallPolicy.Inbound.Region
|
||||
if regionConfig.IsNotEmpty() {
|
||||
for _, remoteAddr := range remoteAddrs {
|
||||
var result = iplib.LookupIP(remoteAddr)
|
||||
if result != nil && result.IsOk() {
|
||||
var currentURL = this.URL()
|
||||
if regionConfig.MatchCountryURL(currentURL) {
|
||||
// 检查国家/地区级别封禁
|
||||
if !regionConfig.IsAllowedCountry(result.CountryId(), result.ProvinceId()) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
if isDefendMode {
|
||||
var promptHTML string
|
||||
if len(regionConfig.CountryHTML) > 0 {
|
||||
promptHTML = regionConfig.CountryHTML
|
||||
@@ -193,23 +196,27 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
|
||||
// 延时返回,避免攻击
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyCountry")
|
||||
}
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyCountry")
|
||||
}
|
||||
|
||||
if isDefendMode {
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if regionConfig.MatchProvinceURL(currentURL) {
|
||||
// 检查省份封禁
|
||||
if !regionConfig.IsAllowedProvince(result.CountryId(), result.ProvinceId()) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
if regionConfig.MatchProvinceURL(currentURL) {
|
||||
// 检查省份封禁
|
||||
if !regionConfig.IsAllowedProvince(result.CountryId(), result.ProvinceId()) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
if isDefendMode {
|
||||
var promptHTML string
|
||||
if len(regionConfig.ProvinceHTML) > 0 {
|
||||
promptHTML = regionConfig.ProvinceHTML
|
||||
@@ -229,14 +236,16 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
|
||||
// 延时返回,避免攻击
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyProvince")
|
||||
}
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyProvince")
|
||||
}
|
||||
|
||||
if isDefendMode {
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
@@ -257,7 +266,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
return
|
||||
}
|
||||
|
||||
goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer)
|
||||
goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer, this.web.FirewallRef.DefaultCaptchaType)
|
||||
if forceLog && logRequestBody && hasRequestBody && ruleSet != nil && ruleSet.HasAttackActions() {
|
||||
this.wafHasRequestBody = true
|
||||
}
|
||||
@@ -307,7 +316,7 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
|
||||
}
|
||||
|
||||
if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
|
||||
blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
|
||||
blocked = this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody, false)
|
||||
if blocked {
|
||||
return true
|
||||
}
|
||||
@@ -315,7 +324,7 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
|
||||
|
||||
// 公用的防火墙设置
|
||||
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
|
||||
blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
|
||||
blocked = this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody, this.web.FirewallRef.IgnoreGlobalRules)
|
||||
if blocked {
|
||||
return true
|
||||
}
|
||||
@@ -469,3 +478,10 @@ func (this *HTTPRequest) WAFMaxRequestSize() int64 {
|
||||
func (this *HTTPRequest) DisableAccessLog() {
|
||||
this.disableLog = true
|
||||
}
|
||||
|
||||
// DisableStat 停用统计
|
||||
func (this *HTTPRequest) DisableStat() {
|
||||
if this.web != nil {
|
||||
this.web.StatRef = nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,9 @@ func (this *HTTPRequest) doWebsocket(requestHost string, isLastRetry bool) (shou
|
||||
}
|
||||
}
|
||||
|
||||
// 标记
|
||||
this.isWebsocketResponse = true
|
||||
|
||||
// 设置指定的来源域
|
||||
if !this.web.Websocket.RequestSameOrigin && len(this.web.Websocket.RequestOrigin) > 0 {
|
||||
var newRequestOrigin = this.web.Websocket.RequestOrigin
|
||||
@@ -77,7 +80,6 @@ func (this *HTTPRequest) doWebsocket(requestHost string, isLastRetry bool) (shou
|
||||
}
|
||||
|
||||
// 连接源站
|
||||
// TODO 增加N次错误重试,重试的时候需要尝试不同的源站
|
||||
originConn, _, err := OriginConnect(this.origin, this.requestServerPort(), this.RawReq.RemoteAddr, requestHost)
|
||||
if err != nil {
|
||||
if isLastRetry {
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
@@ -34,22 +33,19 @@ import (
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
var webpMaxBufferSize int64 = 1_000_000_000
|
||||
var webpTotalBufferSize int64 = 0
|
||||
var webpIgnoreURLSet = setutils.NewFixedSet(131072)
|
||||
var webPThreads int32
|
||||
var webPMaxThreads int32 = 1
|
||||
var webPIgnoreURLSet = setutils.NewFixedSet(131072)
|
||||
|
||||
func init() {
|
||||
if !teaconst.IsMain {
|
||||
return
|
||||
}
|
||||
|
||||
var systemMemory = utils.SystemMemoryGB() / 8
|
||||
if systemMemory > 0 {
|
||||
webpMaxBufferSize = int64(systemMemory) << 30
|
||||
webPMaxThreads = int32(runtime.NumCPU() / 4)
|
||||
if webPMaxThreads < 1 {
|
||||
webPMaxThreads = 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,6 +76,7 @@ type HTTPWriter struct {
|
||||
// WebP
|
||||
webpIsEncoding bool
|
||||
webpOriginContentType string
|
||||
webpQuality int
|
||||
|
||||
// Compression
|
||||
compressionConfig *serverconfigs.HTTPCompressionConfig
|
||||
@@ -483,8 +480,8 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
|
||||
contentTypeWritten = true
|
||||
}
|
||||
|
||||
err := cacheWriter.WriteAt(start, data)
|
||||
if err != nil {
|
||||
writeErr := cacheWriter.WriteAt(start, data)
|
||||
if writeErr != nil {
|
||||
hasError = true
|
||||
this.cacheIsFinished = false
|
||||
}
|
||||
@@ -531,6 +528,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
|
||||
if policy.RequireCache && this.req.cacheRef == nil {
|
||||
return
|
||||
}
|
||||
this.webpQuality = policy.Quality
|
||||
|
||||
// 限制最小和最大尺寸
|
||||
// TODO 需要将reader修改为LimitReader
|
||||
@@ -550,7 +548,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
|
||||
this.req.web.WebP.MatchResponse(contentType, size, filepath.Ext(this.req.Path()), this.req.Format) &&
|
||||
this.req.web.WebP.MatchAccept(this.req.requestHeader("Accept")) {
|
||||
// 检查是否已经因为尺寸过大而忽略
|
||||
if webpIgnoreURLSet.Has(this.req.URL()) {
|
||||
if webPIgnoreURLSet.Has(this.req.URL()) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -560,8 +558,8 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
|
||||
return
|
||||
}
|
||||
|
||||
// 检查内存
|
||||
if atomic.LoadInt64(&webpTotalBufferSize) >= webpMaxBufferSize {
|
||||
// 检查当前是否正在转换
|
||||
if atomic.LoadInt32(&webPThreads) >= webPMaxThreads {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -622,7 +620,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
|
||||
return
|
||||
}
|
||||
|
||||
if this.compressionConfig.Level <= 0 {
|
||||
if this.compressionConfig.Level < 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1020,6 +1018,11 @@ func (this *HTTPWriter) calculateStaleLife() int {
|
||||
func (this *HTTPWriter) finishWebP() {
|
||||
// 处理WebP
|
||||
if this.webpIsEncoding {
|
||||
atomic.AddInt32(&webPThreads, 1)
|
||||
defer func() {
|
||||
atomic.AddInt32(&webPThreads, -1)
|
||||
}()
|
||||
|
||||
var webpCacheWriter caches.Writer
|
||||
|
||||
// 准备WebP Cache
|
||||
@@ -1080,7 +1083,7 @@ func (this *HTTPWriter) finishWebP() {
|
||||
if isGif {
|
||||
gifImage, err = gif.DecodeAll(reader)
|
||||
if gifImage != nil && (gifImage.Config.Width > gowebp.WebPMaxDimension || gifImage.Config.Height > gowebp.WebPMaxDimension) {
|
||||
webpIgnoreURLSet.Push(this.req.URL())
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
} else {
|
||||
@@ -1088,7 +1091,7 @@ func (this *HTTPWriter) finishWebP() {
|
||||
if imageData != nil {
|
||||
var bound = imageData.Bounds()
|
||||
if bound.Max.X > gowebp.WebPMaxDimension || bound.Max.Y > gowebp.WebPMaxDimension {
|
||||
webpIgnoreURLSet.Push(this.req.URL())
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -1096,19 +1099,21 @@ func (this *HTTPWriter) finishWebP() {
|
||||
|
||||
if err != nil {
|
||||
// 发生了错误终止处理
|
||||
webpIgnoreURLSet.Push(this.req.URL())
|
||||
webPIgnoreURLSet.Push(this.req.URL())
|
||||
return
|
||||
}
|
||||
|
||||
var totalBytes = reader.TotalBytes()
|
||||
atomic.AddInt64(&webpTotalBufferSize, totalBytes)
|
||||
defer func() {
|
||||
atomic.AddInt64(&webpTotalBufferSize, -totalBytes)
|
||||
}()
|
||||
|
||||
var f = types.Float32(this.req.web.WebP.Quality)
|
||||
if f > 100 {
|
||||
f = 100
|
||||
var f = types.Float32(this.webpQuality)
|
||||
if f <= 0 || f > 100 {
|
||||
if this.size > (8<<20) || this.size <= 0 {
|
||||
f = 30
|
||||
} else if this.size > (1 << 20) {
|
||||
f = 50
|
||||
} else if this.size > (128 << 10) {
|
||||
f = 60
|
||||
} else {
|
||||
f = 75
|
||||
}
|
||||
}
|
||||
|
||||
if imageData != nil {
|
||||
|
||||
@@ -46,7 +46,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
|
||||
}
|
||||
}
|
||||
|
||||
tlsPolicy, _, err := this.matchSSL(this.helloServerName(clientInfo))
|
||||
tlsPolicy, _, err := this.matchSSL(this.helloServerNames(clientInfo))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
|
||||
}
|
||||
}
|
||||
|
||||
tlsPolicy, cert, err := this.matchSSL(this.helloServerName(clientInfo))
|
||||
tlsPolicy, cert, err := this.matchSSL(this.helloServerNames(clientInfo))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -85,7 +85,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
|
||||
}
|
||||
|
||||
// 根据域名匹配证书
|
||||
func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
|
||||
func (this *BaseListener) matchSSL(domains []string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
|
||||
var group = this.Group
|
||||
|
||||
if group == nil {
|
||||
@@ -99,7 +99,7 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
|
||||
|
||||
// 如果域名为空,则取第一个
|
||||
// 通常域名为空是因为是直接通过IP访问的
|
||||
if len(domain) == 0 {
|
||||
if len(domains) == 0 {
|
||||
if group.IsHTTPS() && globalServerConfig != nil && globalServerConfig.HTTPAll.MatchDomainStrictly {
|
||||
return nil, nil, errors.New("no tls server name matched")
|
||||
}
|
||||
@@ -116,9 +116,25 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
|
||||
}
|
||||
return nil, nil, errors.New("no tls server name found")
|
||||
}
|
||||
var firstDomain = domains[0]
|
||||
|
||||
// 通过网站域名配置匹配
|
||||
server, _ := this.findNamedServer(domain)
|
||||
var server *serverconfigs.ServerConfig
|
||||
var matchedDomain string
|
||||
for _, domain := range domains {
|
||||
server, _ = this.findNamedServer(domain, true)
|
||||
if server != nil {
|
||||
matchedDomain = domain
|
||||
break
|
||||
}
|
||||
}
|
||||
if server == nil {
|
||||
server, _ = this.findNamedServer(firstDomain, false)
|
||||
if server != nil {
|
||||
matchedDomain = firstDomain
|
||||
}
|
||||
}
|
||||
|
||||
if server == nil {
|
||||
// 找不到或者此时的服务没有配置证书,需要搜索所有的Server,通过SSL证书内容中的DNSName匹配
|
||||
// 此功能仅为了兼容以往版本(v1.0.4),不应该作为常态启用
|
||||
@@ -127,14 +143,14 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
|
||||
if searchingServer.SSLPolicy() == nil || !searchingServer.SSLPolicy().IsOn {
|
||||
continue
|
||||
}
|
||||
cert, ok := searchingServer.SSLPolicy().MatchDomain(domain)
|
||||
cert, ok := searchingServer.SSLPolicy().MatchDomain(firstDomain)
|
||||
if ok {
|
||||
return searchingServer.SSLPolicy(), cert, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, errors.New("no server found for '" + domain + "'")
|
||||
return nil, nil, errors.New("no server found for '" + firstDomain + "'")
|
||||
}
|
||||
if server.SSLPolicy() == nil || !server.SSLPolicy().IsOn {
|
||||
// 找不到或者此时的服务没有配置证书,需要搜索所有的Server,通过SSL证书内容中的DNSName匹配
|
||||
@@ -144,32 +160,32 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
|
||||
if searchingServer.SSLPolicy() == nil || !searchingServer.SSLPolicy().IsOn {
|
||||
continue
|
||||
}
|
||||
cert, ok := searchingServer.SSLPolicy().MatchDomain(domain)
|
||||
cert, ok := searchingServer.SSLPolicy().MatchDomain(matchedDomain)
|
||||
if ok {
|
||||
return searchingServer.SSLPolicy(), cert, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, errors.New("no cert found for '" + domain + "'")
|
||||
return nil, nil, errors.New("no cert found for '" + matchedDomain + "'")
|
||||
}
|
||||
|
||||
// 证书是否匹配
|
||||
var sslConfig = server.SSLPolicy()
|
||||
cert, ok := sslConfig.MatchDomain(domain)
|
||||
cert, ok := sslConfig.MatchDomain(matchedDomain)
|
||||
if ok {
|
||||
return sslConfig, cert, nil
|
||||
}
|
||||
|
||||
if len(sslConfig.Certs) == 0 {
|
||||
remotelogs.ServerError(server.Id, "BASE_LISTENER", "no ssl certs found for '"+domain+"', server id: "+types.String(server.Id), "", nil)
|
||||
remotelogs.ServerError(server.Id, "BASE_LISTENER", "no ssl certs found for '"+matchedDomain+"', server id: "+types.String(server.Id), "", nil)
|
||||
}
|
||||
|
||||
return sslConfig, sslConfig.FirstCert(), nil
|
||||
}
|
||||
|
||||
// 根据域名来查找匹配的域名
|
||||
func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconfigs.ServerConfig, serverName string) {
|
||||
func (this *BaseListener) findNamedServer(name string, exactly bool) (serverConfig *serverconfigs.ServerConfig, serverName string) {
|
||||
serverConfig, serverName = this.findNamedServerMatched(name)
|
||||
if serverConfig != nil {
|
||||
return
|
||||
@@ -194,18 +210,22 @@ func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconf
|
||||
}
|
||||
}
|
||||
|
||||
if matchDomainStrictly && !configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) && (!globalServerConfig.HTTPAll.AllowNodeIP || !utils.IsWildIP(name)) {
|
||||
if matchDomainStrictly && !configutils.MatchDomains(globalServerConfig.HTTPAll.AllowMismatchDomains, name) && (!globalServerConfig.HTTPAll.AllowNodeIP || (!utils.IsWildIP(name) || globalServerConfig.HTTPAll.NodeIPShowPage)) {
|
||||
return
|
||||
}
|
||||
|
||||
// 如果没有找到,则匹配到第一个
|
||||
var group = this.Group
|
||||
var currentServers = group.Servers()
|
||||
var countServers = len(currentServers)
|
||||
if countServers == 0 {
|
||||
return nil, ""
|
||||
if !exactly {
|
||||
// 如果没有找到,则匹配到第一个
|
||||
var group = this.Group
|
||||
var currentServers = group.Servers()
|
||||
var countServers = len(currentServers)
|
||||
if countServers == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
return currentServers[0], name
|
||||
}
|
||||
return currentServers[0], name
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// 严格查找域名
|
||||
@@ -234,16 +254,23 @@ func (this *BaseListener) findNamedServerMatched(name string) (serverConfig *ser
|
||||
}
|
||||
|
||||
// 从Hello信息中获取服务名称
|
||||
func (this *BaseListener) helloServerName(clientInfo *tls.ClientHelloInfo) string {
|
||||
var serverName = clientInfo.ServerName
|
||||
if len(serverName) == 0 && clientInfo.Conn != nil {
|
||||
func (this *BaseListener) helloServerNames(clientInfo *tls.ClientHelloInfo) (serverNames []string) {
|
||||
if len(clientInfo.ServerName) != 0 {
|
||||
serverNames = append(serverNames, clientInfo.ServerName)
|
||||
return
|
||||
}
|
||||
|
||||
if clientInfo.Conn != nil {
|
||||
var localAddr = clientInfo.Conn.LocalAddr()
|
||||
if localAddr != nil {
|
||||
tcpAddr, ok := localAddr.(*net.TCPAddr)
|
||||
if ok {
|
||||
serverName = tcpAddr.IP.String()
|
||||
serverNames = append(serverNames, tcpAddr.IP.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
return serverName
|
||||
|
||||
serverNames = append(serverNames, sharedNodeConfig.IPAddresses...)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -107,15 +107,23 @@ func (this *HTTPListener) Reload(group *serverconfigs.ServerAddressGroup) {
|
||||
|
||||
// ServerHTTP 处理HTTP请求
|
||||
func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.Request) {
|
||||
if len(rawReq.Host) > 253 {
|
||||
http.Error(rawWriter, "Host too long.", http.StatusBadRequest)
|
||||
time.Sleep(1 * time.Second) // make connection slow down
|
||||
return
|
||||
}
|
||||
|
||||
var globalServerConfig = sharedNodeConfig.GlobalServerConfig
|
||||
if globalServerConfig != nil && !globalServerConfig.HTTPAll.SupportsLowVersionHTTP && (rawReq.ProtoMajor < 1 /** 0.x **/ || (rawReq.ProtoMajor == 1 && rawReq.ProtoMinor == 0 /** 1.0 **/)) {
|
||||
http.Error(rawWriter, rawReq.Proto+" request is not supported.", http.StatusBadRequest)
|
||||
time.Sleep(1 * time.Second) // make connection slow down
|
||||
return
|
||||
}
|
||||
|
||||
// 不支持Connect
|
||||
if rawReq.Method == http.MethodConnect {
|
||||
http.Error(rawWriter, "Method Not Allowed", http.StatusMethodNotAllowed)
|
||||
time.Sleep(1 * time.Second) // make connection slow down
|
||||
return
|
||||
}
|
||||
|
||||
@@ -154,7 +162,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
|
||||
domain = reqHost
|
||||
}
|
||||
|
||||
server, serverName := this.findNamedServer(domain)
|
||||
server, serverName := this.findNamedServer(domain, false)
|
||||
if server == nil {
|
||||
if server == nil {
|
||||
// 增加默认的一个服务
|
||||
|
||||
@@ -47,9 +47,13 @@ func (this *TCPListener) Serve() error {
|
||||
atomic.AddInt64(&this.countActiveConnections, 1)
|
||||
|
||||
go func(conn net.Conn) {
|
||||
err = this.handleConn(conn)
|
||||
var server = this.Group.FirstServer()
|
||||
if server == nil {
|
||||
return
|
||||
}
|
||||
err = this.handleConn(server, conn)
|
||||
if err != nil {
|
||||
remotelogs.Error("TCP_LISTENER", err.Error())
|
||||
remotelogs.ServerError(server.Id, "TCP_LISTENER", err.Error(), "", nil)
|
||||
}
|
||||
atomic.AddInt64(&this.countActiveConnections, -1)
|
||||
}(conn)
|
||||
@@ -63,8 +67,7 @@ func (this *TCPListener) Reload(group *serverconfigs.ServerAddressGroup) {
|
||||
this.Reset()
|
||||
}
|
||||
|
||||
func (this *TCPListener) handleConn(conn net.Conn) error {
|
||||
var server = this.Group.FirstServer()
|
||||
func (this *TCPListener) handleConn(server *serverconfigs.ServerConfig, conn net.Conn) error {
|
||||
if server == nil {
|
||||
return errors.New("no server available")
|
||||
}
|
||||
@@ -132,14 +135,14 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
|
||||
serverName = tlsConn.ConnectionState().ServerName
|
||||
if len(serverName) > 0 {
|
||||
// 统计
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, serverName, 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, serverName, 0, 0, 1, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
recordStat = true
|
||||
}
|
||||
}
|
||||
|
||||
// 统计
|
||||
if !recordStat {
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
}
|
||||
|
||||
originConn, err := this.connectOrigin(server.Id, serverName, server.ReverseProxy, conn.RemoteAddr().String())
|
||||
@@ -194,7 +197,7 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
|
||||
|
||||
// 记录流量
|
||||
if server != nil {
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -370,7 +370,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyListener
|
||||
|
||||
// 统计
|
||||
if server != nil {
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", 0, 0, 1, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
}
|
||||
|
||||
// 处理ControlMessage
|
||||
@@ -401,7 +401,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyListener
|
||||
// 记录流量和带宽
|
||||
if server != nil {
|
||||
// 流量
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(server.UserId, server.Id, "", int64(n), 0, 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
|
||||
// 带宽
|
||||
var userPlanId int64
|
||||
|
||||
@@ -139,9 +139,6 @@ func (this *Node) Start() {
|
||||
remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
|
||||
}
|
||||
|
||||
// 调整系统参数
|
||||
this.checkSystem()
|
||||
|
||||
// 启动事件
|
||||
events.Notify(events.EventStart)
|
||||
|
||||
@@ -149,12 +146,12 @@ func (this *Node) Start() {
|
||||
remotelogs.Println("NODE", "init config ...")
|
||||
err = this.syncConfig(0)
|
||||
if err != nil {
|
||||
_, err := nodeconfigs.SharedNodeConfig()
|
||||
_, err = nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
// 无本地数据时,会尝试多次读取
|
||||
tryTimes := 0
|
||||
for {
|
||||
err := this.syncConfig(0)
|
||||
err = this.syncConfig(0)
|
||||
if err != nil {
|
||||
tryTimes++
|
||||
|
||||
@@ -208,6 +205,9 @@ func (this *Node) Start() {
|
||||
sharedNodeConfig = nodeConfig
|
||||
this.onReload(nodeConfig, true)
|
||||
|
||||
// 调整系统参数
|
||||
go this.tuneSystemParameters()
|
||||
|
||||
// 发送事件
|
||||
events.Notify(events.EventLoaded)
|
||||
|
||||
@@ -777,9 +777,19 @@ func (this *Node) listenSock() error {
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
case "gc":
|
||||
var before = time.Now()
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
_ = cmd.ReplyOk()
|
||||
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
var gcStats = &debug.GCStats{}
|
||||
debug.ReadGCStats(gcStats)
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]any{
|
||||
"pauseMS": gcStats.PauseTotal.Seconds() * 1000,
|
||||
"costMS": costSeconds * 1000,
|
||||
},
|
||||
})
|
||||
case "reload":
|
||||
err := this.syncConfig(0)
|
||||
if err != nil {
|
||||
@@ -1039,7 +1049,7 @@ func (this *Node) reloadServer() {
|
||||
for serverId, serverConfig := range updatingServerMap {
|
||||
if serverConfig != nil {
|
||||
if countUpdatingServers < maxPrintServers {
|
||||
remotelogs.Debug("NODE", "load server '"+types.String(serverId)+"'")
|
||||
remotelogs.Debug("NODE", "reload server '"+types.String(serverId)+"'")
|
||||
}
|
||||
newNodeConfig.AddServer(serverConfig)
|
||||
} else {
|
||||
@@ -1078,11 +1088,15 @@ func (this *Node) reloadServer() {
|
||||
}
|
||||
|
||||
// 检查系统
|
||||
func (this *Node) checkSystem() {
|
||||
func (this *Node) tuneSystemParameters() {
|
||||
if runtime.GOOS != "linux" || os.Getgid() != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if sharedNodeConfig == nil || !sharedNodeConfig.AutoSystemTuning {
|
||||
return
|
||||
}
|
||||
|
||||
type variable struct {
|
||||
name string
|
||||
minValue int
|
||||
@@ -1091,7 +1105,8 @@ func (this *Node) checkSystem() {
|
||||
|
||||
const dir = "/proc/sys"
|
||||
|
||||
for _, v := range []variable{
|
||||
// net
|
||||
var systemParameters = []variable{
|
||||
{name: "net.core.somaxconn", minValue: 2048},
|
||||
{name: "net.ipv4.tcp_max_syn_backlog", minValue: 2048},
|
||||
{name: "net.core.netdev_max_backlog", minValue: 4096},
|
||||
@@ -1101,7 +1116,28 @@ func (this *Node) checkSystem() {
|
||||
{name: "net.core.wmem_default", minValue: 4 << 20},
|
||||
{name: "net.core.rmem_max", minValue: 32 << 20},
|
||||
{name: "net.core.wmem_max", minValue: 32 << 20},
|
||||
} {
|
||||
}
|
||||
|
||||
// vm
|
||||
var systemMemory = utils.SystemMemoryGB()
|
||||
if systemMemory >= 128 {
|
||||
systemParameters = append(systemParameters, []variable{
|
||||
{name: "vm.dirty_background_ratio", minValue: 40},
|
||||
{name: "vm.dirty_ratio", minValue: 60},
|
||||
}...)
|
||||
} else if systemMemory >= 64 {
|
||||
systemParameters = append(systemParameters, []variable{
|
||||
{name: "vm.dirty_background_ratio", minValue: 30},
|
||||
{name: "vm.dirty_ratio", minValue: 50},
|
||||
}...)
|
||||
} else if systemMemory >= 16 {
|
||||
systemParameters = append(systemParameters, []variable{
|
||||
{name: "vm.dirty_background_ratio", minValue: 15},
|
||||
{name: "vm.dirty_ratio", minValue: 30},
|
||||
}...)
|
||||
}
|
||||
|
||||
for _, v := range systemParameters {
|
||||
var path = dir + "/" + strings.Replace(v.name, ".", "/", -1)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
|
||||
@@ -223,6 +223,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
||||
// 当前TeaWeb所在的fs
|
||||
var rootFS = ""
|
||||
var rootTotal = uint64(0)
|
||||
var totalUsed = uint64(0)
|
||||
if lists.ContainsString([]string{"darwin", "linux", "freebsd"}, runtime.GOOS) {
|
||||
for _, p := range partitions {
|
||||
if p.Mountpoint == "/" {
|
||||
@@ -230,6 +231,7 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
||||
usage, _ := disk.Usage(p.Mountpoint)
|
||||
if usage != nil {
|
||||
rootTotal = usage.Total
|
||||
totalUsed = usage.Used
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -237,7 +239,6 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
||||
}
|
||||
|
||||
var total = rootTotal
|
||||
var totalUsage = uint64(0)
|
||||
var maxUsage = float64(0)
|
||||
for _, partition := range partitions {
|
||||
if runtime.GOOS != "windows" && !strings.Contains(partition.Device, "/") && !strings.Contains(partition.Device, "\\") {
|
||||
@@ -256,16 +257,16 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
|
||||
|
||||
if partition.Mountpoint != "/" && (usage.Total != rootTotal || total == 0) {
|
||||
total += usage.Total
|
||||
}
|
||||
totalUsage += usage.Used
|
||||
if usage.UsedPercent >= maxUsage {
|
||||
maxUsage = usage.UsedPercent
|
||||
status.DiskMaxUsagePartition = partition.Mountpoint
|
||||
totalUsed += usage.Used
|
||||
if usage.UsedPercent >= maxUsage {
|
||||
maxUsage = usage.UsedPercent
|
||||
status.DiskMaxUsagePartition = partition.Mountpoint
|
||||
}
|
||||
}
|
||||
}
|
||||
status.DiskTotal = total
|
||||
if total > 0 {
|
||||
status.DiskUsage = float64(totalUsage) / float64(total)
|
||||
status.DiskUsage = float64(totalUsed) / float64(total)
|
||||
}
|
||||
status.DiskMaxUsage = maxUsage / 100
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
@@ -97,6 +98,10 @@ func (this *Node) execTask(rpcClient *rpc.RPCClient, task *pb.NodeTask) error {
|
||||
err = this.notifyPlusChange()
|
||||
case "toaChanged":
|
||||
err = this.execTOAChangedTask()
|
||||
case "networkSecurityPolicyChanged":
|
||||
err = this.execNetworkSecurityPolicyChangedTask(rpcClient)
|
||||
case "webPPolicyChanged":
|
||||
err = this.execWebPPolicyChangedTask(rpcClient)
|
||||
default:
|
||||
// 特殊任务
|
||||
if strings.HasPrefix(task.Type, "ipListDeleted") { // 删除IP名单
|
||||
@@ -296,7 +301,7 @@ func (this *Node) execUpdatingServersTask(rpcClient *rpc.RPCClient) error {
|
||||
|
||||
// 删除IP名单
|
||||
func (this *Node) execDeleteIPList(taskType string) error {
|
||||
optionsString, ok := strings.CutPrefix(taskType, "ipListDeleted@")
|
||||
optionsString, ok := utils.CutPrefix(taskType, "ipListDeleted@")
|
||||
if !ok {
|
||||
return errors.New("invalid task type '" + taskType + "'")
|
||||
}
|
||||
@@ -322,6 +327,34 @@ func (this *Node) execDeleteIPList(taskType string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// WebP策略变更
|
||||
func (this *Node) execWebPPolicyChangedTask(rpcClient *rpc.RPCClient) error {
|
||||
remotelogs.Println("NODE", "updating webp policies ...")
|
||||
resp, err := rpcClient.NodeRPC.FindNodeWebPPolicies(rpcClient.Context(), &pb.FindNodeWebPPoliciesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var webPPolicyMap = map[int64]*nodeconfigs.WebPImagePolicy{}
|
||||
for _, policy := range resp.WebPPolicies {
|
||||
if len(policy.WebPPolicyJSON) > 0 {
|
||||
var webPPolicy = nodeconfigs.NewWebPImagePolicy()
|
||||
err = json.Unmarshal(policy.WebPPolicyJSON, webPPolicy)
|
||||
if err != nil {
|
||||
remotelogs.Error("NODE", "decode webp policy failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
err = webPPolicy.Init()
|
||||
if err != nil {
|
||||
remotelogs.Error("NODE", "initialize webp policy failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
webPPolicyMap[policy.NodeClusterId] = webPPolicy
|
||||
}
|
||||
}
|
||||
sharedNodeConfig.UpdateWebPImagePolicies(webPPolicyMap)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 标记任务完成
|
||||
func (this *Node) finishTask(taskId int64, taskVersion int64, taskErr error) (success bool) {
|
||||
if taskId <= 0 {
|
||||
|
||||
@@ -29,3 +29,8 @@ func (this *Node) execHTTPPagesPolicyChangedTask(rpcClient *rpc.RPCClient) error
|
||||
// stub
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Node) execNetworkSecurityPolicyChangedTask(rpcClient *rpc.RPCClient) error {
|
||||
// stub
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ package re
|
||||
|
||||
type RuneMap map[rune]*RuneTree
|
||||
|
||||
func (this *RuneMap) Lookup(s string, caseInsensitive bool) bool {
|
||||
func (this RuneMap) Lookup(s string, caseInsensitive bool) bool {
|
||||
return this.lookup([]rune(s), caseInsensitive, 0)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/encoding/gzip"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"net/url"
|
||||
"sync"
|
||||
@@ -240,12 +241,15 @@ func (this *RPCClient) init() error {
|
||||
grpc.MaxCallSendMsgSize(512<<20),
|
||||
grpc.UseCompressor(gzip.Name),
|
||||
)
|
||||
var keepaliveParams = grpc.WithKeepaliveParams(keepalive.ClientParameters{
|
||||
Time: 30 * time.Second,
|
||||
})
|
||||
if u.Scheme == "http" {
|
||||
conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(insecure.NewCredentials()), callOptions)
|
||||
conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(insecure.NewCredentials()), callOptions, keepaliveParams)
|
||||
} else if u.Scheme == "https" {
|
||||
conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
})), callOptions)
|
||||
})), callOptions, keepaliveParams)
|
||||
} else {
|
||||
return errors.New("parse endpoint failed: invalid scheme '" + u.Scheme + "'")
|
||||
}
|
||||
|
||||
@@ -57,12 +57,13 @@ type BandwidthStat struct {
|
||||
MaxBytes int64 `json:"maxBytes"`
|
||||
TotalBytes int64 `json:"totalBytes"`
|
||||
|
||||
CachedBytes int64 `json:"cachedBytes"`
|
||||
AttackBytes int64 `json:"attackBytes"`
|
||||
CountRequests int64 `json:"countRequests"`
|
||||
CountCachedRequests int64 `json:"countCachedRequests"`
|
||||
CountAttackRequests int64 `json:"countAttackRequests"`
|
||||
UserPlanId int64 `json:"userPlanId"`
|
||||
CachedBytes int64 `json:"cachedBytes"`
|
||||
AttackBytes int64 `json:"attackBytes"`
|
||||
CountRequests int64 `json:"countRequests"`
|
||||
CountCachedRequests int64 `json:"countCachedRequests"`
|
||||
CountAttackRequests int64 `json:"countAttackRequests"`
|
||||
CountWebsocketConnections int64 `json:"countWebsocketConnections"`
|
||||
UserPlanId int64 `json:"userPlanId"`
|
||||
}
|
||||
|
||||
// BandwidthStatManager 服务带宽统计
|
||||
@@ -142,20 +143,21 @@ func (this *BandwidthStatManager) Loop() error {
|
||||
}
|
||||
|
||||
pbStats = append(pbStats, &pb.ServerBandwidthStat{
|
||||
Id: 0,
|
||||
UserId: stat.UserId,
|
||||
ServerId: stat.ServerId,
|
||||
Day: stat.Day,
|
||||
TimeAt: stat.TimeAt,
|
||||
Bytes: stat.MaxBytes / bandwidthTimestampDelim,
|
||||
TotalBytes: stat.TotalBytes,
|
||||
CachedBytes: stat.CachedBytes,
|
||||
AttackBytes: stat.AttackBytes,
|
||||
CountRequests: stat.CountRequests,
|
||||
CountCachedRequests: stat.CountCachedRequests,
|
||||
CountAttackRequests: stat.CountAttackRequests,
|
||||
UserPlanId: stat.UserPlanId,
|
||||
NodeRegionId: regionId,
|
||||
Id: 0,
|
||||
UserId: stat.UserId,
|
||||
ServerId: stat.ServerId,
|
||||
Day: stat.Day,
|
||||
TimeAt: stat.TimeAt,
|
||||
Bytes: stat.MaxBytes / bandwidthTimestampDelim,
|
||||
TotalBytes: stat.TotalBytes,
|
||||
CachedBytes: stat.CachedBytes,
|
||||
AttackBytes: stat.AttackBytes,
|
||||
CountRequests: stat.CountRequests,
|
||||
CountCachedRequests: stat.CountCachedRequests,
|
||||
CountAttackRequests: stat.CountAttackRequests,
|
||||
CountWebsocketConnections: stat.CountWebsocketConnections,
|
||||
UserPlanId: stat.UserPlanId,
|
||||
NodeRegionId: regionId,
|
||||
})
|
||||
delete(this.m, key)
|
||||
}
|
||||
@@ -231,7 +233,7 @@ func (this *BandwidthStatManager) AddBandwidth(userId int64, userPlanId int64, s
|
||||
}
|
||||
|
||||
// AddTraffic 添加请求数据
|
||||
func (this *BandwidthStatManager) AddTraffic(serverId int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64) {
|
||||
func (this *BandwidthStatManager) AddTraffic(serverId int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, countWebsocketConnections int64) {
|
||||
var now = fasttime.Now()
|
||||
var day = now.Ymd()
|
||||
var timeAt = now.Round5Hi()
|
||||
@@ -245,6 +247,7 @@ func (this *BandwidthStatManager) AddTraffic(serverId int64, cachedBytes int64,
|
||||
stat.CountCachedRequests += countCachedRequests
|
||||
stat.CountAttackRequests += countAttacks
|
||||
stat.AttackBytes += attackBytes
|
||||
stat.CountWebsocketConnections += countWebsocketConnections
|
||||
}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
@@ -53,19 +53,20 @@ func BenchmarkBandwidthStatManager_Slice(b *testing.B) {
|
||||
for j := 0; j < 100; j++ {
|
||||
var stat = &stats.BandwidthStat{}
|
||||
pbStats = append(pbStats, &pb.ServerBandwidthStat{
|
||||
Id: 0,
|
||||
UserId: stat.UserId,
|
||||
ServerId: stat.ServerId,
|
||||
Day: stat.Day,
|
||||
TimeAt: stat.TimeAt,
|
||||
Bytes: stat.MaxBytes / 2,
|
||||
TotalBytes: stat.TotalBytes,
|
||||
CachedBytes: stat.CachedBytes,
|
||||
AttackBytes: stat.AttackBytes,
|
||||
CountRequests: stat.CountRequests,
|
||||
CountCachedRequests: stat.CountCachedRequests,
|
||||
CountAttackRequests: stat.CountAttackRequests,
|
||||
NodeRegionId: 1,
|
||||
Id: 0,
|
||||
UserId: stat.UserId,
|
||||
ServerId: stat.ServerId,
|
||||
Day: stat.Day,
|
||||
TimeAt: stat.TimeAt,
|
||||
Bytes: stat.MaxBytes / 2,
|
||||
TotalBytes: stat.TotalBytes,
|
||||
CachedBytes: stat.CachedBytes,
|
||||
AttackBytes: stat.AttackBytes,
|
||||
CountRequests: stat.CountRequests,
|
||||
CountCachedRequests: stat.CountCachedRequests,
|
||||
CountAttackRequests: stat.CountAttackRequests,
|
||||
CountWebsocketConnections: stat.CountWebsocketConnections,
|
||||
NodeRegionId: 1,
|
||||
})
|
||||
}
|
||||
_ = pbStats
|
||||
|
||||
@@ -106,13 +106,13 @@ func (this *TrafficStatManager) Start() {
|
||||
}
|
||||
|
||||
// Add 添加流量
|
||||
func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, checkingTrafficLimit bool, planId int64) {
|
||||
func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, countWebsocketConnections int64, checkingTrafficLimit bool, planId int64) {
|
||||
if serverId == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// 添加到带宽
|
||||
SharedBandwidthStatManager.AddTraffic(serverId, cachedBytes, countRequests, countCachedRequests, countAttacks, attackBytes)
|
||||
SharedBandwidthStatManager.AddTraffic(serverId, cachedBytes, countRequests, countCachedRequests, countAttacks, attackBytes, countWebsocketConnections)
|
||||
|
||||
if bytes == 0 && countRequests == 0 {
|
||||
return
|
||||
@@ -142,24 +142,26 @@ func (this *TrafficStatManager) Add(userId int64, serverId int64, domain string,
|
||||
item.PlanId = planId
|
||||
|
||||
// 单个域名流量
|
||||
var domainKey = types.String(timestamp) + "@" + domain
|
||||
serverDomainMap, ok := this.domainsMap[serverId]
|
||||
if !ok {
|
||||
serverDomainMap = map[string]*TrafficItem{}
|
||||
this.domainsMap[serverId] = serverDomainMap
|
||||
}
|
||||
if len(domain) < 128 {
|
||||
var domainKey = types.String(timestamp) + "@" + domain
|
||||
serverDomainMap, ok := this.domainsMap[serverId]
|
||||
if !ok {
|
||||
serverDomainMap = map[string]*TrafficItem{}
|
||||
this.domainsMap[serverId] = serverDomainMap
|
||||
}
|
||||
|
||||
domainItem, ok := serverDomainMap[domainKey]
|
||||
if !ok {
|
||||
domainItem = &TrafficItem{}
|
||||
serverDomainMap[domainKey] = domainItem
|
||||
domainItem, ok := serverDomainMap[domainKey]
|
||||
if !ok {
|
||||
domainItem = &TrafficItem{}
|
||||
serverDomainMap[domainKey] = domainItem
|
||||
}
|
||||
domainItem.Bytes += bytes
|
||||
domainItem.CachedBytes += cachedBytes
|
||||
domainItem.CountRequests += countRequests
|
||||
domainItem.CountCachedRequests += countCachedRequests
|
||||
domainItem.CountAttackRequests += countAttacks
|
||||
domainItem.AttackBytes += attackBytes
|
||||
}
|
||||
domainItem.Bytes += bytes
|
||||
domainItem.CachedBytes += cachedBytes
|
||||
domainItem.CountRequests += countRequests
|
||||
domainItem.CountCachedRequests += countCachedRequests
|
||||
domainItem.CountAttackRequests += countAttacks
|
||||
domainItem.AttackBytes += attackBytes
|
||||
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
func TestTrafficStatManager_Add(t *testing.T) {
|
||||
manager := NewTrafficStatManager()
|
||||
for i := 0; i < 100; i++ {
|
||||
manager.Add(1, 1, "goedge.cn", 1, 0, 0, 0, 0, 0, false, 0)
|
||||
manager.Add(1, 1, "goedge.cn", 1, 0, 0, 0, 0, 0, 0, false, 0)
|
||||
}
|
||||
t.Log(manager.itemMap)
|
||||
}
|
||||
@@ -19,7 +19,7 @@ func TestTrafficStatManager_Add(t *testing.T) {
|
||||
func TestTrafficStatManager_Upload(t *testing.T) {
|
||||
manager := NewTrafficStatManager()
|
||||
for i := 0; i < 100; i++ {
|
||||
manager.Add(1, 1, "goedge.cn"+types.String(rands.Int(0, 10)), 1, 0, 1, 0, 0, 0, false, 0)
|
||||
manager.Add(1, 1, "goedge.cn"+types.String(rands.Int(0, 10)), 1, 0, 1, 0, 0, 0, 0, false, 0)
|
||||
}
|
||||
err := manager.Upload()
|
||||
if err != nil {
|
||||
@@ -36,7 +36,7 @@ func BenchmarkTrafficStatManager_Add(b *testing.B) {
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
manager.Add(1, 1, "goedge.cn"+types.String(rand.Int63()%10), 1024, 1, 0, 0, 0, 0, false, 0)
|
||||
manager.Add(1, 1, "goedge.cn"+types.String(rand.Int63()%10), 1024, 1, 0, 0, 0, 0, 0, false, 0)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,9 +3,10 @@ package ttlcache
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
var SharedCache = NewBigCache()
|
||||
var SharedInt64Cache = NewBigCache[int64]()
|
||||
|
||||
// Cache TTL缓存
|
||||
// 最大的缓存时间为30 * 86400
|
||||
@@ -13,24 +14,25 @@ var SharedCache = NewBigCache()
|
||||
//
|
||||
// Piece1 | Piece2 | Piece3 | ...
|
||||
// [ Item1, Item2, ... ] | ...
|
||||
type Cache struct {
|
||||
type Cache[T any] struct {
|
||||
isDestroyed bool
|
||||
pieces []*Piece
|
||||
pieces []*Piece[T]
|
||||
countPieces uint64
|
||||
maxItems int
|
||||
|
||||
gcPieceIndex int
|
||||
maxPiecesPerGC int
|
||||
gcPieceIndex int
|
||||
}
|
||||
|
||||
func NewBigCache() *Cache {
|
||||
func NewBigCache[T any]() *Cache[T] {
|
||||
var delta = utils.SystemMemoryGB() / 2
|
||||
if delta <= 0 {
|
||||
delta = 1
|
||||
}
|
||||
return NewCache(NewMaxItemsOption(delta * 1_000_000))
|
||||
return NewCache[T](NewMaxItemsOption(delta * 1_000_000))
|
||||
}
|
||||
|
||||
func NewCache(opt ...OptionInterface) *Cache {
|
||||
func NewCache[T any](opt ...OptionInterface) *Cache[T] {
|
||||
var countPieces = 256
|
||||
var maxItems = 1_000_000
|
||||
|
||||
@@ -61,13 +63,20 @@ func NewCache(opt ...OptionInterface) *Cache {
|
||||
}
|
||||
}
|
||||
|
||||
var cache = &Cache{
|
||||
countPieces: uint64(countPieces),
|
||||
maxItems: maxItems,
|
||||
var maxPiecesPerGC = 4
|
||||
var numCPU = runtime.NumCPU() / 2
|
||||
if numCPU > maxPiecesPerGC {
|
||||
maxPiecesPerGC = numCPU
|
||||
}
|
||||
|
||||
var cache = &Cache[T]{
|
||||
countPieces: uint64(countPieces),
|
||||
maxItems: maxItems,
|
||||
maxPiecesPerGC: maxPiecesPerGC,
|
||||
}
|
||||
|
||||
for i := 0; i < countPieces; i++ {
|
||||
cache.pieces = append(cache.pieces, NewPiece(maxItems/countPieces))
|
||||
cache.pieces = append(cache.pieces, NewPiece[T](maxItems/countPieces))
|
||||
}
|
||||
|
||||
// Add to manager
|
||||
@@ -76,7 +85,7 @@ func NewCache(opt ...OptionInterface) *Cache {
|
||||
return cache
|
||||
}
|
||||
|
||||
func (this *Cache) Write(key string, value any, expiredAt int64) (ok bool) {
|
||||
func (this *Cache[T]) Write(key string, value T, expiredAt int64) (ok bool) {
|
||||
if this.isDestroyed {
|
||||
return
|
||||
}
|
||||
@@ -92,20 +101,20 @@ func (this *Cache) Write(key string, value any, expiredAt int64) (ok bool) {
|
||||
}
|
||||
var uint64Key = HashKey([]byte(key))
|
||||
var pieceIndex = uint64Key % this.countPieces
|
||||
return this.pieces[pieceIndex].Add(uint64Key, &Item{
|
||||
return this.pieces[pieceIndex].Add(uint64Key, &Item[T]{
|
||||
Value: value,
|
||||
expiredAt: expiredAt,
|
||||
})
|
||||
}
|
||||
|
||||
func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64, extend bool) int64 {
|
||||
func (this *Cache[T]) IncreaseInt64(key string, delta T, expiredAt int64, extend bool) T {
|
||||
if this.isDestroyed {
|
||||
return 0
|
||||
return any(0).(T)
|
||||
}
|
||||
|
||||
var currentTimestamp = fasttime.Now().Unix()
|
||||
if expiredAt <= currentTimestamp {
|
||||
return 0
|
||||
return any(0).(T)
|
||||
}
|
||||
|
||||
var maxExpiredAt = currentTimestamp + 30*86400
|
||||
@@ -117,47 +126,47 @@ func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64, exten
|
||||
return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiredAt, extend)
|
||||
}
|
||||
|
||||
func (this *Cache) Read(key string) (item *Item) {
|
||||
func (this *Cache[T]) Read(key string) (item *Item[T]) {
|
||||
var uint64Key = HashKey([]byte(key))
|
||||
return this.pieces[uint64Key%this.countPieces].Read(uint64Key)
|
||||
}
|
||||
|
||||
func (this *Cache) Delete(key string) {
|
||||
func (this *Cache[T]) Delete(key string) {
|
||||
var uint64Key = HashKey([]byte(key))
|
||||
this.pieces[uint64Key%this.countPieces].Delete(uint64Key)
|
||||
}
|
||||
|
||||
func (this *Cache) Count() (count int) {
|
||||
func (this *Cache[T]) Count() (count int) {
|
||||
for _, piece := range this.pieces {
|
||||
count += piece.Count()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Cache) GC() {
|
||||
func (this *Cache[T]) GC() {
|
||||
var index = this.gcPieceIndex
|
||||
const maxPiecesPerGC = 4
|
||||
for i := index; i < index+maxPiecesPerGC; i++ {
|
||||
|
||||
for i := index; i < index+this.maxPiecesPerGC; i++ {
|
||||
if i >= int(this.countPieces) {
|
||||
break
|
||||
}
|
||||
this.pieces[i].GC()
|
||||
}
|
||||
|
||||
index += maxPiecesPerGC
|
||||
index += this.maxPiecesPerGC
|
||||
if index >= int(this.countPieces) {
|
||||
index = 0
|
||||
}
|
||||
this.gcPieceIndex = index
|
||||
}
|
||||
|
||||
func (this *Cache) Clean() {
|
||||
func (this *Cache[T]) Clean() {
|
||||
for _, piece := range this.pieces {
|
||||
piece.Clean()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Cache) Destroy() {
|
||||
func (this *Cache[T]) Destroy() {
|
||||
SharedManager.Remove(this)
|
||||
|
||||
this.isDestroyed = true
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
func TestNewCache(t *testing.T) {
|
||||
var cache = NewCache()
|
||||
var cache = NewCache[int]()
|
||||
cache.Write("a", 1, time.Now().Unix()+3600)
|
||||
cache.Write("b", 2, time.Now().Unix()+1)
|
||||
cache.Write("c", 1, time.Now().Unix()+3602)
|
||||
@@ -28,7 +29,9 @@ func TestNewCache(t *testing.T) {
|
||||
}
|
||||
}
|
||||
t.Log("a:", cache.Read("a"))
|
||||
time.Sleep(5 * time.Second)
|
||||
if testutils.IsSingleTesting() {
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
for i := 0; i < len(cache.pieces); i++ {
|
||||
cache.GC()
|
||||
@@ -40,12 +43,19 @@ func TestNewCache(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCache_Memory(t *testing.T) {
|
||||
testutils.StartMemoryStats(t)
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var cache = NewCache()
|
||||
var count = 2_000_000
|
||||
var cache = NewCache[int]()
|
||||
|
||||
testutils.StartMemoryStats(t, func() {
|
||||
t.Log(cache.Count(), "items")
|
||||
})
|
||||
|
||||
var count = 20_000_000
|
||||
for i := 0; i < count; i++ {
|
||||
cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+3600)
|
||||
cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+int64(rands.Int(0, 300)))
|
||||
}
|
||||
|
||||
t.Log(cache.Count())
|
||||
@@ -61,27 +71,27 @@ func TestCache_Memory(t *testing.T) {
|
||||
|
||||
cache.Count()
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
time.Sleep(3600 * time.Second)
|
||||
}
|
||||
|
||||
func TestCache_IncreaseInt64(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var cache = NewCache()
|
||||
var cache = NewCache[int64]()
|
||||
var unixTime = time.Now().Unix()
|
||||
|
||||
{
|
||||
cache.IncreaseInt64("a", 1, unixTime+3600, false)
|
||||
var item = cache.Read("a")
|
||||
t.Log(item)
|
||||
a.IsTrue(item.Value == int64(1))
|
||||
a.IsTrue(item.Value == 1)
|
||||
a.IsTrue(item.expiredAt == unixTime+3600)
|
||||
}
|
||||
{
|
||||
cache.IncreaseInt64("a", 1, unixTime+3600+1, true)
|
||||
var item = cache.Read("a")
|
||||
t.Log(item)
|
||||
a.IsTrue(item.Value == int64(2))
|
||||
a.IsTrue(item.Value == 2)
|
||||
a.IsTrue(item.expiredAt == unixTime+3600+1)
|
||||
}
|
||||
{
|
||||
@@ -97,7 +107,7 @@ func TestCache_IncreaseInt64(t *testing.T) {
|
||||
func TestCache_Read(t *testing.T) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache = NewCache(PiecesOption{Count: 32})
|
||||
var cache = NewCache[int](PiecesOption{Count: 32})
|
||||
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
cache.Write("HELLO_WORLD_"+strconv.Itoa(i), i, time.Now().Unix()+int64(i%10240)+1)
|
||||
@@ -119,7 +129,11 @@ func TestCache_Read(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCache_GC(t *testing.T) {
|
||||
var cache = NewCache(&PiecesOption{Count: 5})
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var cache = NewCache[int](&PiecesOption{Count: 5})
|
||||
cache.Write("a", 1, time.Now().Unix()+1)
|
||||
cache.Write("b", 2, time.Now().Unix()+2)
|
||||
cache.Write("c", 3, time.Now().Unix()+3)
|
||||
@@ -153,26 +167,30 @@ func TestCache_GC(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCache_GC2(t *testing.T) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache1 = NewCache(NewPiecesOption(32))
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
cache1.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(0, 10)))
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var cache2 = NewCache(NewPiecesOption(5))
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache1 = NewCache[int](NewPiecesOption(256))
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
cache1.Write(strconv.Itoa(i), i, time.Now().Unix()+10)
|
||||
}
|
||||
|
||||
var cache2 = NewCache[int](NewPiecesOption(5))
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
cache2.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(0, 10)))
|
||||
}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
t.Log(cache1.Count(), "items", cache2.Count(), "items")
|
||||
for i := 0; i < 3600; i++ {
|
||||
t.Log(timeutil.Format("H:i:s"), cache1.Count(), "items", cache2.Count(), "items")
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheDestroy(t *testing.T) {
|
||||
var cache = NewCache()
|
||||
var cache = NewCache[int]()
|
||||
t.Log("count:", SharedManager.Count())
|
||||
cache.Destroy()
|
||||
t.Log("count:", SharedManager.Count())
|
||||
@@ -181,7 +199,7 @@ func TestCacheDestroy(t *testing.T) {
|
||||
func BenchmarkNewCache(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache = NewCache(NewPiecesOption(128))
|
||||
var cache = NewCache[int](NewPiecesOption(128))
|
||||
for i := 0; i < 2_000_000; i++ {
|
||||
cache.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(10, 100)))
|
||||
}
|
||||
@@ -199,7 +217,7 @@ func BenchmarkNewCache(b *testing.B) {
|
||||
func BenchmarkCache_Add(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache = NewCache()
|
||||
var cache = NewCache[int]()
|
||||
for i := 0; i < b.N; i++ {
|
||||
cache.Write(strconv.Itoa(i), i, fasttime.Now().Unix()+int64(i%1024))
|
||||
}
|
||||
@@ -208,7 +226,7 @@ func BenchmarkCache_Add(b *testing.B) {
|
||||
func BenchmarkCache_Add_Parallel(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache = NewCache()
|
||||
var cache = NewCache[int64]()
|
||||
var i int64
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
@@ -221,7 +239,7 @@ func BenchmarkCache_Add_Parallel(b *testing.B) {
|
||||
func BenchmarkNewCacheGC(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache = NewCache(NewPiecesOption(1024))
|
||||
var cache = NewCache[int](NewPiecesOption(1024))
|
||||
for i := 0; i < 3_000_000; i++ {
|
||||
cache.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(0, 100)))
|
||||
}
|
||||
@@ -238,7 +256,7 @@ func BenchmarkNewCacheGC(b *testing.B) {
|
||||
func BenchmarkNewCacheClean(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var cache = NewCache(NewPiecesOption(128))
|
||||
var cache = NewCache[int](NewPiecesOption(128))
|
||||
for i := 0; i < 3_000_000; i++ {
|
||||
cache.Write(strconv.Itoa(i), i, time.Now().Unix()+int64(rands.Int(10, 100)))
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package ttlcache
|
||||
|
||||
type Item struct {
|
||||
Value any
|
||||
type Item[T any] struct {
|
||||
Value T
|
||||
expiredAt int64
|
||||
}
|
||||
|
||||
@@ -11,17 +11,21 @@ import (
|
||||
|
||||
var SharedManager = NewManager()
|
||||
|
||||
type GCAble interface {
|
||||
GC()
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
ticker *time.Ticker
|
||||
locker sync.Mutex
|
||||
|
||||
cacheMap map[*Cache]zero.Zero
|
||||
cacheMap map[GCAble]zero.Zero
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
var manager = &Manager{
|
||||
ticker: time.NewTicker(2 * time.Second),
|
||||
cacheMap: map[*Cache]zero.Zero{},
|
||||
cacheMap: map[GCAble]zero.Zero{},
|
||||
}
|
||||
|
||||
goman.New(func() {
|
||||
@@ -41,13 +45,13 @@ func (this *Manager) init() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Manager) Add(cache *Cache) {
|
||||
func (this *Manager) Add(cache GCAble) {
|
||||
this.locker.Lock()
|
||||
this.cacheMap[cache] = zero.New()
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *Manager) Remove(cache *Cache) {
|
||||
func (this *Manager) Remove(cache GCAble) {
|
||||
this.locker.Lock()
|
||||
delete(this.cacheMap, cache)
|
||||
this.locker.Unlock()
|
||||
|
||||
@@ -3,12 +3,11 @@ package ttlcache
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/expires"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Piece struct {
|
||||
m map[uint64]*Item
|
||||
type Piece[T any] struct {
|
||||
m map[uint64]*Item[T]
|
||||
expiresList *expires.List
|
||||
maxItems int
|
||||
lastGCTime int64
|
||||
@@ -16,20 +15,28 @@ type Piece struct {
|
||||
locker sync.RWMutex
|
||||
}
|
||||
|
||||
func NewPiece(maxItems int) *Piece {
|
||||
return &Piece{
|
||||
m: map[uint64]*Item{},
|
||||
func NewPiece[T any](maxItems int) *Piece[T] {
|
||||
return &Piece[T]{
|
||||
m: map[uint64]*Item[T]{},
|
||||
expiresList: expires.NewSingletonList(),
|
||||
maxItems: maxItems,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Piece) Add(key uint64, item *Item) (ok bool) {
|
||||
this.locker.Lock()
|
||||
func (this *Piece[T]) Add(key uint64, item *Item[T]) (ok bool) {
|
||||
this.locker.RLock()
|
||||
if this.maxItems > 0 && len(this.m) >= this.maxItems {
|
||||
this.locker.Unlock()
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
|
||||
this.locker.Lock()
|
||||
oldItem, exists := this.m[key]
|
||||
if exists && oldItem.expiredAt == item.expiredAt {
|
||||
this.locker.Unlock()
|
||||
return true
|
||||
}
|
||||
this.m[key] = item
|
||||
this.locker.Unlock()
|
||||
|
||||
@@ -38,11 +45,14 @@ func (this *Piece) Add(key uint64, item *Item) (ok bool) {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, extend bool) (result int64) {
|
||||
func (this *Piece[T]) IncreaseInt64(key uint64, delta T, expiredAt int64, extend bool) (result T) {
|
||||
this.locker.Lock()
|
||||
item, ok := this.m[key]
|
||||
if ok && item.expiredAt > fasttime.Now().Unix() {
|
||||
result = types.Int64(item.Value) + delta
|
||||
int64Value, isInt64 := any(item.Value).(int64)
|
||||
if isInt64 {
|
||||
result = any(int64Value + any(delta).(int64)).(T)
|
||||
}
|
||||
item.Value = result
|
||||
if extend {
|
||||
item.expiredAt = expiredAt
|
||||
@@ -51,7 +61,7 @@ func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, exten
|
||||
} else {
|
||||
if len(this.m) < this.maxItems {
|
||||
result = delta
|
||||
this.m[key] = &Item{
|
||||
this.m[key] = &Item[T]{
|
||||
Value: delta,
|
||||
expiredAt: expiredAt,
|
||||
}
|
||||
@@ -63,7 +73,7 @@ func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, exten
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Piece) Delete(key uint64) {
|
||||
func (this *Piece[T]) Delete(key uint64) {
|
||||
this.expiresList.Remove(key)
|
||||
|
||||
this.locker.Lock()
|
||||
@@ -71,7 +81,7 @@ func (this *Piece) Delete(key uint64) {
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *Piece) Read(key uint64) (item *Item) {
|
||||
func (this *Piece[T]) Read(key uint64) (item *Item[T]) {
|
||||
this.locker.RLock()
|
||||
item = this.m[key]
|
||||
if item != nil && item.expiredAt < fasttime.Now().Unix() {
|
||||
@@ -82,27 +92,27 @@ func (this *Piece) Read(key uint64) (item *Item) {
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Piece) Count() (count int) {
|
||||
func (this *Piece[T]) Count() (count int) {
|
||||
this.locker.RLock()
|
||||
count = len(this.m)
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Piece) GC() {
|
||||
func (this *Piece[T]) GC() {
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
if this.lastGCTime == 0 {
|
||||
this.lastGCTime = currentTime - 3600
|
||||
}
|
||||
|
||||
var min = this.lastGCTime
|
||||
var max = currentTime
|
||||
if min > max {
|
||||
var minTime = this.lastGCTime
|
||||
var maxTime = currentTime
|
||||
if minTime > maxTime {
|
||||
// 过去的时间比现在大,则从这一秒重新开始
|
||||
min = max
|
||||
minTime = maxTime
|
||||
}
|
||||
|
||||
for i := min; i <= max; i++ {
|
||||
for i := minTime; i <= maxTime; i++ {
|
||||
var itemMap = this.expiresList.GC(i)
|
||||
if len(itemMap) > 0 {
|
||||
this.gcItemMap(itemMap)
|
||||
@@ -112,15 +122,15 @@ func (this *Piece) GC() {
|
||||
this.lastGCTime = currentTime
|
||||
}
|
||||
|
||||
func (this *Piece) Clean() {
|
||||
func (this *Piece[T]) Clean() {
|
||||
this.locker.Lock()
|
||||
this.m = map[uint64]*Item{}
|
||||
this.m = map[uint64]*Item[T]{}
|
||||
this.locker.Unlock()
|
||||
|
||||
this.expiresList.Clean()
|
||||
}
|
||||
|
||||
func (this *Piece) Destroy() {
|
||||
func (this *Piece[T]) Destroy() {
|
||||
this.locker.Lock()
|
||||
this.m = nil
|
||||
this.locker.Unlock()
|
||||
@@ -128,7 +138,7 @@ func (this *Piece) Destroy() {
|
||||
this.expiresList.Clean()
|
||||
}
|
||||
|
||||
func (this *Piece) gcItemMap(itemMap expires.ItemMap) {
|
||||
func (this *Piece[T]) gcItemMap(itemMap expires.ItemMap) {
|
||||
this.locker.Lock()
|
||||
for key := range itemMap {
|
||||
delete(this.m, key)
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
)
|
||||
|
||||
func TestPiece_Add(t *testing.T) {
|
||||
piece := NewPiece(10)
|
||||
piece.Add(1, &Item{expiredAt: time.Now().Unix() + 3600})
|
||||
piece.Add(2, &Item{})
|
||||
piece.Add(3, &Item{})
|
||||
piece := NewPiece[int](10)
|
||||
piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600})
|
||||
piece.Add(2, &Item[int]{})
|
||||
piece.Add(3, &Item[int]{})
|
||||
piece.Delete(3)
|
||||
for key, item := range piece.m {
|
||||
t.Log(key, item.Value)
|
||||
@@ -18,19 +18,29 @@ func TestPiece_Add(t *testing.T) {
|
||||
t.Log(piece.Read(1))
|
||||
}
|
||||
|
||||
func TestPiece_Add_Same(t *testing.T) {
|
||||
piece := NewPiece[int](10)
|
||||
piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600})
|
||||
piece.Add(1, &Item[int]{expiredAt: time.Now().Unix() + 3600})
|
||||
for key, item := range piece.m {
|
||||
t.Log(key, item.Value)
|
||||
}
|
||||
t.Log(piece.Read(1))
|
||||
}
|
||||
|
||||
func TestPiece_MaxItems(t *testing.T) {
|
||||
piece := NewPiece(10)
|
||||
piece := NewPiece[int](10)
|
||||
for i := 0; i < 1000; i++ {
|
||||
piece.Add(uint64(i), &Item{expiredAt: time.Now().Unix() + 3600})
|
||||
piece.Add(uint64(i), &Item[int]{expiredAt: time.Now().Unix() + 3600})
|
||||
}
|
||||
t.Log(len(piece.m))
|
||||
}
|
||||
|
||||
func TestPiece_GC(t *testing.T) {
|
||||
piece := NewPiece(10)
|
||||
piece.Add(1, &Item{Value: 1, expiredAt: time.Now().Unix() + 1})
|
||||
piece.Add(2, &Item{Value: 2, expiredAt: time.Now().Unix() + 1})
|
||||
piece.Add(3, &Item{Value: 3, expiredAt: time.Now().Unix() + 1})
|
||||
piece := NewPiece[int](10)
|
||||
piece.Add(1, &Item[int]{Value: 1, expiredAt: time.Now().Unix() + 1})
|
||||
piece.Add(2, &Item[int]{Value: 2, expiredAt: time.Now().Unix() + 1})
|
||||
piece.Add(3, &Item[int]{Value: 3, expiredAt: time.Now().Unix() + 1})
|
||||
t.Log("before gc ===")
|
||||
for key, item := range piece.m {
|
||||
t.Log(key, item.Value)
|
||||
@@ -46,9 +56,9 @@ func TestPiece_GC(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPiece_GC2(t *testing.T) {
|
||||
piece := NewPiece(10)
|
||||
piece := NewPiece[int](10)
|
||||
for i := 0; i < 10_000; i++ {
|
||||
piece.Add(uint64(i), &Item{Value: 1, expiredAt: time.Now().Unix() + int64(rands.Int(1, 10))})
|
||||
piece.Add(uint64(i), &Item[int]{Value: 1, expiredAt: time.Now().Unix() + int64(rands.Int(1, 10))})
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
162
internal/utils/cachehits/stat.go
Normal file
162
internal/utils/cachehits/stat.go
Normal file
@@ -0,0 +1,162 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package cachehits
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const countSamples = 100_000
|
||||
|
||||
type Item struct {
|
||||
countHits uint64
|
||||
countCached uint64
|
||||
timestamp int64
|
||||
|
||||
isGood bool
|
||||
isBad bool
|
||||
}
|
||||
|
||||
type Stat struct {
|
||||
goodRatio uint64
|
||||
maxItems int
|
||||
|
||||
itemMap map[string]*Item // category => *Item
|
||||
mu *sync.RWMutex
|
||||
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
func NewStat(goodRatio uint64) *Stat {
|
||||
if goodRatio == 0 {
|
||||
goodRatio = 5
|
||||
}
|
||||
|
||||
var maxItems = utils.SystemMemoryGB() * 10_000
|
||||
if maxItems <= 0 {
|
||||
maxItems = 100_000
|
||||
}
|
||||
|
||||
var stat = &Stat{
|
||||
goodRatio: goodRatio,
|
||||
itemMap: map[string]*Item{},
|
||||
mu: &sync.RWMutex{},
|
||||
ticker: time.NewTicker(24 * time.Hour),
|
||||
maxItems: maxItems,
|
||||
}
|
||||
|
||||
goman.New(func() {
|
||||
stat.init()
|
||||
})
|
||||
return stat
|
||||
}
|
||||
|
||||
func (this *Stat) init() {
|
||||
for range this.ticker.C {
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
|
||||
this.mu.RLock()
|
||||
for _, item := range this.itemMap {
|
||||
if item.timestamp < currentTime-7*24*86400 {
|
||||
// reset
|
||||
item.countHits = 0
|
||||
item.countCached = 1
|
||||
item.timestamp = currentTime
|
||||
item.isGood = false
|
||||
item.isBad = false
|
||||
}
|
||||
}
|
||||
this.mu.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stat) IncreaseCached(category string) {
|
||||
this.mu.RLock()
|
||||
var item = this.itemMap[category]
|
||||
if item != nil {
|
||||
if item.isGood || item.isBad {
|
||||
this.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
atomic.AddUint64(&item.countCached, 1)
|
||||
this.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
this.mu.RUnlock()
|
||||
|
||||
this.mu.Lock()
|
||||
|
||||
if len(this.itemMap) > this.maxItems {
|
||||
// remove one randomly
|
||||
for k := range this.itemMap {
|
||||
delete(this.itemMap, k)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
this.itemMap[category] = &Item{
|
||||
countHits: 0,
|
||||
countCached: 1,
|
||||
timestamp: fasttime.Now().Unix(),
|
||||
}
|
||||
this.mu.Unlock()
|
||||
}
|
||||
|
||||
func (this *Stat) IncreaseHit(category string) {
|
||||
this.mu.RLock()
|
||||
defer this.mu.RUnlock()
|
||||
|
||||
var item = this.itemMap[category]
|
||||
if item != nil {
|
||||
if item.isGood || item.isBad {
|
||||
return
|
||||
}
|
||||
|
||||
atomic.AddUint64(&item.countHits, 1)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Stat) IsGood(category string) bool {
|
||||
this.mu.RLock()
|
||||
defer func() {
|
||||
this.mu.RUnlock()
|
||||
}()
|
||||
|
||||
var item = this.itemMap[category]
|
||||
if item != nil {
|
||||
if item.isBad {
|
||||
return false
|
||||
}
|
||||
if item.isGood {
|
||||
return true
|
||||
}
|
||||
|
||||
if item.countCached > countSamples && (Tea.IsTesting() || item.timestamp < fasttime.Now().Unix()-600) /** 10 minutes ago **/ {
|
||||
var isGood = item.countHits*100/item.countCached >= this.goodRatio
|
||||
if isGood {
|
||||
item.isGood = true
|
||||
} else {
|
||||
item.isBad = true
|
||||
}
|
||||
|
||||
return isGood
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *Stat) Len() int {
|
||||
this.mu.RLock()
|
||||
defer this.mu.RUnlock()
|
||||
|
||||
return len(this.itemMap)
|
||||
}
|
||||
107
internal/utils/cachehits/stat_test.go
Normal file
107
internal/utils/cachehits/stat_test.go
Normal file
@@ -0,0 +1,107 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package cachehits_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/cachehits"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewStat(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
var stat = cachehits.NewStat(20)
|
||||
for i := 0; i < 1000; i++ {
|
||||
stat.IncreaseCached("a")
|
||||
}
|
||||
|
||||
a.IsTrue(stat.IsGood("a"))
|
||||
}
|
||||
|
||||
{
|
||||
var stat = cachehits.NewStat(5)
|
||||
for i := 0; i < 10000; i++ {
|
||||
stat.IncreaseCached("a")
|
||||
}
|
||||
for i := 0; i < 500; i++ {
|
||||
stat.IncreaseHit("a")
|
||||
}
|
||||
|
||||
stat.IncreaseHit("b") // empty
|
||||
|
||||
a.IsTrue(stat.IsGood("a"))
|
||||
a.IsTrue(stat.IsGood("b"))
|
||||
}
|
||||
|
||||
{
|
||||
var stat = cachehits.NewStat(10)
|
||||
for i := 0; i < 10000; i++ {
|
||||
stat.IncreaseCached("a")
|
||||
}
|
||||
for i := 0; i < 1000; i++ {
|
||||
stat.IncreaseHit("a")
|
||||
}
|
||||
|
||||
stat.IncreaseHit("b") // empty
|
||||
|
||||
a.IsTrue(stat.IsGood("a"))
|
||||
a.IsTrue(stat.IsGood("b"))
|
||||
}
|
||||
|
||||
{
|
||||
var stat = cachehits.NewStat(5)
|
||||
for i := 0; i < 100001; i++ {
|
||||
stat.IncreaseCached("a")
|
||||
}
|
||||
for i := 0; i < 4999; i++ {
|
||||
stat.IncreaseHit("a")
|
||||
}
|
||||
|
||||
a.IsFalse(stat.IsGood("a"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewStat_Memory(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var stat = cachehits.NewStat(20)
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
stat.IncreaseCached("a" + types.String(i))
|
||||
}
|
||||
|
||||
time.Sleep(60 * time.Second)
|
||||
|
||||
t.Log(stat.Len())
|
||||
}
|
||||
|
||||
func BenchmarkStat(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var stat = cachehits.NewStat(5)
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
stat.IncreaseCached("a" + types.String(i))
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var key = strconv.Itoa(rands.Int(0, 100_000))
|
||||
stat.IncreaseCached(key)
|
||||
if rands.Int(0, 3) == 0 {
|
||||
stat.IncreaseHit(key)
|
||||
}
|
||||
_ = stat.IsGood(key)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -3,18 +3,26 @@
|
||||
package counters
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
syncutils "github.com/TeaOSLab/EdgeNode/internal/utils/sync"
|
||||
"github.com/cespare/xxhash"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Counter struct {
|
||||
const maxItemsPerGroup = 50_000
|
||||
|
||||
var SharedCounter = NewCounter[uint32]().WithGC()
|
||||
|
||||
type SupportedUIntType interface {
|
||||
uint32 | uint64
|
||||
}
|
||||
|
||||
type Counter[T SupportedUIntType] struct {
|
||||
countMaps uint64
|
||||
locker *syncutils.RWMutex
|
||||
itemMaps []map[uint64]*Item
|
||||
itemMaps []map[uint64]*Item[T]
|
||||
|
||||
gcTicker *time.Ticker
|
||||
gcIndex int
|
||||
@@ -22,20 +30,18 @@ type Counter struct {
|
||||
}
|
||||
|
||||
// NewCounter create new counter
|
||||
func NewCounter() *Counter {
|
||||
var count = runtime.NumCPU() * 4
|
||||
func NewCounter[T SupportedUIntType]() *Counter[T] {
|
||||
var count = utils.SystemMemoryGB() * 8
|
||||
if count < 8 {
|
||||
count = 8
|
||||
} else if count > 128 {
|
||||
count = 128
|
||||
}
|
||||
|
||||
var itemMaps = []map[uint64]*Item{}
|
||||
var itemMaps = []map[uint64]*Item[T]{}
|
||||
for i := 0; i < count; i++ {
|
||||
itemMaps = append(itemMaps, map[uint64]*Item{})
|
||||
itemMaps = append(itemMaps, map[uint64]*Item[T]{})
|
||||
}
|
||||
|
||||
var counter = &Counter{
|
||||
var counter = &Counter[T]{
|
||||
countMaps: uint64(count),
|
||||
locker: syncutils.NewRWMutex(count),
|
||||
itemMaps: itemMaps,
|
||||
@@ -45,7 +51,7 @@ func NewCounter() *Counter {
|
||||
}
|
||||
|
||||
// WithGC start the counter with gc automatically
|
||||
func (this *Counter) WithGC() *Counter {
|
||||
func (this *Counter[T]) WithGC() *Counter[T] {
|
||||
if this.gcTicker != nil {
|
||||
return this
|
||||
}
|
||||
@@ -60,23 +66,17 @@ func (this *Counter) WithGC() *Counter {
|
||||
}
|
||||
|
||||
// Increase key
|
||||
func (this *Counter) Increase(key uint64, lifeSeconds int) uint64 {
|
||||
func (this *Counter[T]) Increase(key uint64, lifeSeconds int) T {
|
||||
var index = int(key % this.countMaps)
|
||||
this.locker.RLock(index)
|
||||
var item = this.itemMaps[index][key]
|
||||
this.locker.RUnlock(index)
|
||||
if item == nil { // no need to care about duplication
|
||||
item = NewItem(lifeSeconds)
|
||||
if item == nil {
|
||||
// no need to care about duplication
|
||||
// always insert new item even when itemMap is full
|
||||
item = NewItem[T](lifeSeconds)
|
||||
this.locker.Lock(index)
|
||||
|
||||
// check again
|
||||
oldItem, ok := this.itemMaps[index][key]
|
||||
if !ok {
|
||||
this.itemMaps[index][key] = item
|
||||
} else {
|
||||
item = oldItem
|
||||
}
|
||||
|
||||
this.itemMaps[index][key] = item
|
||||
this.locker.Unlock(index)
|
||||
}
|
||||
|
||||
@@ -87,12 +87,12 @@ func (this *Counter) Increase(key uint64, lifeSeconds int) uint64 {
|
||||
}
|
||||
|
||||
// IncreaseKey increase string key
|
||||
func (this *Counter) IncreaseKey(key string, lifeSeconds int) uint64 {
|
||||
func (this *Counter[T]) IncreaseKey(key string, lifeSeconds int) T {
|
||||
return this.Increase(this.hash(key), lifeSeconds)
|
||||
}
|
||||
|
||||
// Get value of key
|
||||
func (this *Counter) Get(key uint64) uint64 {
|
||||
func (this *Counter[T]) Get(key uint64) T {
|
||||
var index = int(key % this.countMaps)
|
||||
this.locker.RLock(index)
|
||||
defer this.locker.RUnlock(index)
|
||||
@@ -104,12 +104,12 @@ func (this *Counter) Get(key uint64) uint64 {
|
||||
}
|
||||
|
||||
// GetKey get value of string key
|
||||
func (this *Counter) GetKey(key string) uint64 {
|
||||
func (this *Counter[T]) GetKey(key string) T {
|
||||
return this.Get(this.hash(key))
|
||||
}
|
||||
|
||||
// Reset key
|
||||
func (this *Counter) Reset(key uint64) {
|
||||
func (this *Counter[T]) Reset(key uint64) {
|
||||
var index = int(key % this.countMaps)
|
||||
this.locker.RLock(index)
|
||||
var item = this.itemMaps[index][key]
|
||||
@@ -123,12 +123,12 @@ func (this *Counter) Reset(key uint64) {
|
||||
}
|
||||
|
||||
// ResetKey string key
|
||||
func (this *Counter) ResetKey(key string) {
|
||||
func (this *Counter[T]) ResetKey(key string) {
|
||||
this.Reset(this.hash(key))
|
||||
}
|
||||
|
||||
// TotalItems get items count
|
||||
func (this *Counter) TotalItems() int {
|
||||
func (this *Counter[T]) TotalItems() int {
|
||||
var total = 0
|
||||
|
||||
for i := 0; i < int(this.countMaps); i++ {
|
||||
@@ -141,7 +141,7 @@ func (this *Counter) TotalItems() int {
|
||||
}
|
||||
|
||||
// GC garbage expired items
|
||||
func (this *Counter) GC() {
|
||||
func (this *Counter[T]) GC() {
|
||||
this.gcLocker.Lock()
|
||||
var gcIndex = this.gcIndex
|
||||
|
||||
@@ -162,18 +162,39 @@ func (this *Counter) GC() {
|
||||
expiredKeys = append(expiredKeys, key)
|
||||
}
|
||||
}
|
||||
var tooManyItems = len(itemMap) > maxItemsPerGroup // prevent too many items
|
||||
this.locker.RUnlock(gcIndex)
|
||||
|
||||
if len(expiredKeys) > 0 {
|
||||
this.locker.Lock(gcIndex)
|
||||
for _, key := range expiredKeys {
|
||||
this.locker.Lock(gcIndex)
|
||||
delete(itemMap, key)
|
||||
this.locker.Unlock(gcIndex)
|
||||
}
|
||||
this.locker.Unlock(gcIndex)
|
||||
}
|
||||
|
||||
if tooManyItems {
|
||||
this.locker.Lock(gcIndex)
|
||||
var count = len(itemMap) - maxItemsPerGroup
|
||||
if count > 0 {
|
||||
itemMap = this.itemMaps[gcIndex]
|
||||
for key := range itemMap {
|
||||
delete(itemMap, key)
|
||||
count--
|
||||
if count < 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
this.locker.Unlock(gcIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Counter[T]) CountMaps() int {
|
||||
return int(this.countMaps)
|
||||
}
|
||||
|
||||
// calculate hash of the key
|
||||
func (this *Counter) hash(key string) uint64 {
|
||||
func (this *Counter[T]) hash(key string) uint64 {
|
||||
return xxhash.Sum64String(key)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
func TestCounter_Increase(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
a.IsTrue(counter.Increase(1, 10) == 1)
|
||||
a.IsTrue(counter.Increase(1, 10) == 2)
|
||||
a.IsTrue(counter.Increase(2, 10) == 1)
|
||||
@@ -31,7 +32,7 @@ func TestCounter_Increase(t *testing.T) {
|
||||
func TestCounter_IncreaseKey(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
a.IsTrue(counter.IncreaseKey("1", 10) == 1)
|
||||
a.IsTrue(counter.IncreaseKey("1", 10) == 2)
|
||||
a.IsTrue(counter.IncreaseKey("2", 10) == 1)
|
||||
@@ -46,13 +47,14 @@ func TestCounter_GC(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
counter.Increase(1, 20)
|
||||
time.Sleep(1 * time.Second)
|
||||
counter.Increase(1, 20)
|
||||
time.Sleep(1 * time.Second)
|
||||
counter.Increase(1, 20)
|
||||
counter.GC()
|
||||
t.Log(counter.Get(1))
|
||||
}
|
||||
|
||||
func TestCounter_GC2(t *testing.T) {
|
||||
@@ -60,8 +62,8 @@ func TestCounter_GC2(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var counter = counters.NewCounter().WithGC()
|
||||
for i := 0; i < 1e5; i++ {
|
||||
var counter = counters.NewCounter[uint32]().WithGC()
|
||||
for i := 0; i < 100_000; i++ {
|
||||
counter.Increase(uint64(i), rands.Int(10, 300))
|
||||
}
|
||||
|
||||
@@ -78,20 +80,39 @@ func TestCounterMemory(t *testing.T) {
|
||||
var stat = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
for i := 0; i < 1e5; i++ {
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
counter.Increase(uint64(i), rands.Int(10, 300))
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
t.Log((stat1.TotalAlloc-stat.TotalAlloc)/(1<<20), "MB")
|
||||
t.Log((stat1.HeapInuse-stat.HeapInuse)/(1<<20), "MB")
|
||||
|
||||
t.Log(counter.TotalItems())
|
||||
|
||||
var gcPause = func() {
|
||||
var before = time.Now()
|
||||
runtime.GC()
|
||||
var costSeconds = time.Since(before).Seconds()
|
||||
var stats = &debug.GCStats{}
|
||||
debug.ReadGCStats(stats)
|
||||
t.Log("GC pause:", stats.PauseTotal.Seconds()*1000, "ms", "cost:", costSeconds*1000, "ms")
|
||||
}
|
||||
|
||||
gcPause()
|
||||
|
||||
_ = counter.TotalItems()
|
||||
}
|
||||
|
||||
func BenchmarkCounter_Increase(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
b.ResetTimer()
|
||||
|
||||
var i uint64
|
||||
@@ -107,7 +128,7 @@ func BenchmarkCounter_Increase(b *testing.B) {
|
||||
func BenchmarkCounter_IncreaseKey(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
|
||||
go func() {
|
||||
var ticker = time.NewTicker(100 * time.Millisecond)
|
||||
@@ -131,7 +152,7 @@ func BenchmarkCounter_IncreaseKey(b *testing.B) {
|
||||
func BenchmarkCounter_IncreaseKey2(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
|
||||
go func() {
|
||||
var ticker = time.NewTicker(1 * time.Millisecond)
|
||||
@@ -155,7 +176,7 @@ func BenchmarkCounter_IncreaseKey2(b *testing.B) {
|
||||
func BenchmarkCounter_GC(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var counter = counters.NewCounter()
|
||||
var counter = counters.NewCounter[uint32]()
|
||||
|
||||
for i := uint64(0); i < 1e5; i++ {
|
||||
counter.IncreaseKey(types.String(i), 20)
|
||||
|
||||
@@ -6,72 +6,123 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
lifeSeconds int64
|
||||
|
||||
spanSeconds int64
|
||||
spans []*Span
|
||||
const spanMaxValue = 10_000_000
|
||||
const maxSpans = 10
|
||||
|
||||
type Item[T SupportedUIntType] struct {
|
||||
spans [maxSpans + 1]T
|
||||
lastUpdateTime int64
|
||||
lifeSeconds int64
|
||||
spanSeconds int64
|
||||
}
|
||||
|
||||
func NewItem(lifeSeconds int) *Item {
|
||||
func NewItem[T SupportedUIntType](lifeSeconds int) *Item[T] {
|
||||
if lifeSeconds <= 0 {
|
||||
lifeSeconds = 60
|
||||
}
|
||||
var spanSeconds = lifeSeconds / 10
|
||||
var spanSeconds = lifeSeconds / maxSpans
|
||||
if spanSeconds < 1 {
|
||||
spanSeconds = 1
|
||||
}
|
||||
var countSpans = lifeSeconds/spanSeconds + 1 /** prevent index out of bounds **/
|
||||
var spans = []*Span{}
|
||||
for i := 0; i < countSpans; i++ {
|
||||
spans = append(spans, NewSpan())
|
||||
} else if lifeSeconds > maxSpans && lifeSeconds%maxSpans != 0 {
|
||||
spanSeconds++
|
||||
}
|
||||
|
||||
return &Item{
|
||||
return &Item[T]{
|
||||
lifeSeconds: int64(lifeSeconds),
|
||||
spanSeconds: int64(spanSeconds),
|
||||
spans: spans,
|
||||
lastUpdateTime: fasttime.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Item) Increase() uint64 {
|
||||
func (this *Item[T]) Increase() (result T) {
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
var spanIndex = int(currentTime % this.lifeSeconds / this.spanSeconds)
|
||||
var span = this.spans[spanIndex]
|
||||
var roundTime = currentTime / this.spanSeconds * this.spanSeconds
|
||||
var currentSpanIndex = this.calculateSpanIndex(currentTime)
|
||||
|
||||
this.lastUpdateTime = currentTime
|
||||
|
||||
if span.Timestamp < roundTime {
|
||||
span.Timestamp = roundTime // update time
|
||||
span.Count = 0 // reset count
|
||||
// return quickly
|
||||
if this.lastUpdateTime == currentTime {
|
||||
if this.spans[currentSpanIndex] < spanMaxValue {
|
||||
this.spans[currentSpanIndex]++
|
||||
}
|
||||
for _, count := range this.spans {
|
||||
result += count
|
||||
}
|
||||
return
|
||||
}
|
||||
span.Count++
|
||||
|
||||
return this.Sum()
|
||||
}
|
||||
if this.lastUpdateTime > 0 {
|
||||
if currentTime-this.lastUpdateTime > this.lifeSeconds {
|
||||
for index := range this.spans {
|
||||
this.spans[index] = 0
|
||||
}
|
||||
} else {
|
||||
var lastSpanIndex = this.calculateSpanIndex(this.lastUpdateTime)
|
||||
|
||||
func (this *Item) Sum() uint64 {
|
||||
var result uint64 = 0
|
||||
var currentTimestamp = fasttime.Now().Unix()
|
||||
for _, span := range this.spans {
|
||||
if span.Timestamp >= currentTimestamp-this.lifeSeconds {
|
||||
result += span.Count
|
||||
if lastSpanIndex != currentSpanIndex {
|
||||
var countSpans = len(this.spans)
|
||||
|
||||
// reset values between LAST and CURRENT
|
||||
for index := lastSpanIndex + 1; ; index++ {
|
||||
var realIndex = index % countSpans
|
||||
this.spans[realIndex] = 0
|
||||
if realIndex == currentSpanIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if this.spans[currentSpanIndex] < spanMaxValue {
|
||||
this.spans[currentSpanIndex]++
|
||||
}
|
||||
this.lastUpdateTime = currentTime
|
||||
|
||||
for _, count := range this.spans {
|
||||
result += count
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Item[T]) Sum() (result T) {
|
||||
if this.lastUpdateTime == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var currentTime = fasttime.Now().Unix()
|
||||
var currentSpanIndex = this.calculateSpanIndex(currentTime)
|
||||
|
||||
if currentTime-this.lastUpdateTime > this.lifeSeconds {
|
||||
return 0
|
||||
} else {
|
||||
var lastSpanIndex = this.calculateSpanIndex(this.lastUpdateTime)
|
||||
var countSpans = len(this.spans)
|
||||
for index := currentSpanIndex + 1; ; index++ {
|
||||
var realIndex = index % countSpans
|
||||
result += this.spans[realIndex]
|
||||
if realIndex == lastSpanIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (this *Item) Reset() {
|
||||
for _, span := range this.spans {
|
||||
span.Count = 0
|
||||
span.Timestamp = 0
|
||||
func (this *Item[T]) Reset() {
|
||||
for index := range this.spans {
|
||||
this.spans[index] = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Item) IsExpired(currentTime int64) bool {
|
||||
func (this *Item[T]) IsExpired(currentTime int64) bool {
|
||||
return this.lastUpdateTime < currentTime-this.lifeSeconds-this.spanSeconds
|
||||
}
|
||||
|
||||
func (this *Item[T]) calculateSpanIndex(timestamp int64) int {
|
||||
var index = int(timestamp % this.lifeSeconds / this.spanSeconds)
|
||||
if index > maxSpans-1 {
|
||||
return maxSpans - 1
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/counters"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"runtime"
|
||||
"testing"
|
||||
@@ -13,6 +14,27 @@ import (
|
||||
)
|
||||
|
||||
func TestItem_Increase(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var item = counters.NewItem[uint32](10)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
time.Sleep(1 * time.Second)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
time.Sleep(2 * time.Second)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
time.Sleep(5 * time.Second)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
time.Sleep(6 * time.Second)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
time.Sleep(5 * time.Second)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
time.Sleep(11 * time.Second)
|
||||
t.Log(item.Increase(), item.Sum())
|
||||
}
|
||||
|
||||
func TestItem_Increase2(t *testing.T) {
|
||||
// run only under single testing
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
@@ -20,9 +42,9 @@ func TestItem_Increase(t *testing.T) {
|
||||
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var item = counters.NewItem(20)
|
||||
var item = counters.NewItem[uint32](23)
|
||||
for i := 0; i < 100; i++ {
|
||||
t.Log(item.Increase(), timeutil.Format("i:s"))
|
||||
t.Log("round "+types.String(i)+":", item.Increase(), item.Sum(), timeutil.Format("H:i:s"))
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
@@ -35,22 +57,26 @@ func TestItem_IsExpired(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
var currentTime = time.Now().Unix()
|
||||
|
||||
var item = counters.NewItem(10)
|
||||
t.Log(item.IsExpired(currentTime))
|
||||
var item = counters.NewItem[uint32](10)
|
||||
t.Log(item.IsExpired(time.Now().Unix()))
|
||||
time.Sleep(10 * time.Second)
|
||||
t.Log(item.IsExpired(currentTime))
|
||||
t.Log(item.IsExpired(time.Now().Unix()))
|
||||
time.Sleep(2 * time.Second)
|
||||
t.Log(item.IsExpired(currentTime))
|
||||
t.Log(item.IsExpired(time.Now().Unix()))
|
||||
time.Sleep(2 * time.Second)
|
||||
t.Log(item.IsExpired(time.Now().Unix()))
|
||||
}
|
||||
|
||||
func BenchmarkItem_Increase(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var item = counters.NewItem(60)
|
||||
b.ReportAllocs()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
item.Increase()
|
||||
}
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var item = counters.NewItem[uint32](60)
|
||||
item.Increase()
|
||||
item.Sum()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package counters
|
||||
|
||||
type Span struct {
|
||||
Timestamp int64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
func NewSpan() *Span {
|
||||
return &Span{}
|
||||
}
|
||||
@@ -18,6 +18,10 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
SyncMode = "OFF"
|
||||
)
|
||||
|
||||
var errDBIsClosed = errors.New("the database is closed")
|
||||
|
||||
type DB struct {
|
||||
@@ -199,6 +203,7 @@ func (this *DB) Close() error {
|
||||
this.statusLocker.Unlock()
|
||||
|
||||
// waiting for updating operations to finish
|
||||
var maxLoops = 5_000
|
||||
for {
|
||||
this.statusLocker.Lock()
|
||||
var countUpdating = this.countUpdating
|
||||
@@ -207,6 +212,11 @@ func (this *DB) Close() error {
|
||||
break
|
||||
}
|
||||
time.Sleep(1 * time.Millisecond)
|
||||
|
||||
maxLoops--
|
||||
if maxLoops <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, batch := range this.batches {
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
package dbs_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseDSN(t *testing.T) {
|
||||
var dsn = "file:/home/cache/p43/.indexes/db-3.db?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=88000"
|
||||
var dsn = "file:/home/cache/p43/.indexes/db-3.db?cache=private&mode=ro&_journal_mode=WAL&_sync=" + dbs.SyncMode + "&_cache_size=88000"
|
||||
u, err := url.Parse(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -5,6 +5,7 @@ package dbs
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
)
|
||||
|
||||
type Stmt struct {
|
||||
@@ -27,7 +28,7 @@ func (this *Stmt) EnableStat() {
|
||||
this.enableStat = true
|
||||
}
|
||||
|
||||
func (this *Stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, error) {
|
||||
func (this *Stmt) ExecContext(ctx context.Context, args ...any) (result sql.Result, err error) {
|
||||
// check database status
|
||||
if this.db.BeginUpdating() {
|
||||
defer this.db.EndUpdating()
|
||||
@@ -38,10 +39,13 @@ func (this *Stmt) ExecContext(ctx context.Context, args ...any) (sql.Result, err
|
||||
if this.enableStat {
|
||||
defer SharedQueryStatManager.AddQuery(this.query).End()
|
||||
}
|
||||
return this.rawStmt.ExecContext(ctx, args...)
|
||||
fsutils.WriteBegin()
|
||||
result, err = this.rawStmt.ExecContext(ctx, args...)
|
||||
fsutils.WriteEnd()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Stmt) Exec(args ...any) (sql.Result, error) {
|
||||
func (this *Stmt) Exec(args ...any) (result sql.Result, err error) {
|
||||
// check database status
|
||||
if this.db.BeginUpdating() {
|
||||
defer this.db.EndUpdating()
|
||||
@@ -52,7 +56,11 @@ func (this *Stmt) Exec(args ...any) (sql.Result, error) {
|
||||
if this.enableStat {
|
||||
defer SharedQueryStatManager.AddQuery(this.query).End()
|
||||
}
|
||||
return this.rawStmt.Exec(args...)
|
||||
|
||||
fsutils.WriteBegin()
|
||||
result, err = this.rawStmt.Exec(args...)
|
||||
fsutils.WriteEnd()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Stmt) QueryContext(ctx context.Context, args ...any) (*sql.Rows, error) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package expires
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
@@ -84,6 +85,10 @@ func TestList_GC_Batch(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestList_Start_GC(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
list := NewList()
|
||||
list.Add(1, time.Now().Unix()+1)
|
||||
list.Add(2, time.Now().Unix()+1)
|
||||
@@ -125,6 +130,25 @@ func TestList_ManyItems(t *testing.T) {
|
||||
t.Log(time.Since(now))
|
||||
}
|
||||
|
||||
func TestList_Memory(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var list = NewList()
|
||||
|
||||
testutils.StartMemoryStats(t, func() {
|
||||
t.Log(list.Count(), "items")
|
||||
})
|
||||
|
||||
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
list.Add(uint64(i), time.Now().Unix()+1800)
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Hour)
|
||||
}
|
||||
|
||||
func TestList_Map_Performance(t *testing.T) {
|
||||
t.Log("max uint32", math.MaxUint32)
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ func (this *Manager) init() {
|
||||
for range this.ticker.C {
|
||||
var currentTime = time.Now().Unix()
|
||||
if lastTimestamp == 0 {
|
||||
lastTimestamp = currentTime - 3600
|
||||
lastTimestamp = currentTime - 86400 // prevent timezone changes
|
||||
}
|
||||
|
||||
if currentTime >= lastTimestamp {
|
||||
|
||||
@@ -4,11 +4,20 @@ package fsutils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"math"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
const diskSpeedDataFile = "disk.speed.json"
|
||||
|
||||
type DiskSpeedCache struct {
|
||||
Speed Speed `json:"speed"`
|
||||
SpeedMB float64 `json:"speedMB"`
|
||||
}
|
||||
|
||||
// CheckDiskWritingSpeed test disk writing speed
|
||||
func CheckDiskWritingSpeed() (speedMB float64, err error) {
|
||||
var tempDir = os.TempDir()
|
||||
@@ -66,8 +75,13 @@ func CheckDiskIsFast() (speedMB float64, isFast bool, err error) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
isFast = speedMB > 150
|
||||
|
||||
if speedMB <= DiskSpeedMB {
|
||||
return
|
||||
}
|
||||
|
||||
if speedMB > 1000 {
|
||||
DiskSpeed = SpeedExtremelyFast
|
||||
} else if speedMB > 150 {
|
||||
@@ -79,8 +93,15 @@ func CheckDiskIsFast() (speedMB float64, isFast bool, err error) {
|
||||
}
|
||||
calculateDiskMaxWrites()
|
||||
|
||||
if speedMB > DiskSpeedMB {
|
||||
DiskSpeedMB = speedMB
|
||||
DiskSpeedMB = speedMB
|
||||
|
||||
// write to local file
|
||||
cacheData, jsonErr := json.Marshal(&DiskSpeedCache{
|
||||
Speed: DiskSpeed,
|
||||
SpeedMB: DiskSpeedMB,
|
||||
})
|
||||
if jsonErr == nil {
|
||||
_ = os.WriteFile(Tea.Root+"/data/"+diskSpeedDataFile, cacheData, 0666)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@@ -3,7 +3,11 @@
|
||||
package fsutils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/shirou/gopsutil/v3/load"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
@@ -37,6 +41,14 @@ var (
|
||||
DiskSpeedMB float64
|
||||
)
|
||||
|
||||
var IsInHighLoad = false
|
||||
var IsInExtremelyHighLoad = false
|
||||
|
||||
const (
|
||||
highLoad1Threshold = 20
|
||||
extremelyHighLoad1Threshold = 40
|
||||
)
|
||||
|
||||
func init() {
|
||||
if !teaconst.IsMain {
|
||||
return
|
||||
@@ -44,6 +56,18 @@ func init() {
|
||||
|
||||
// test disk
|
||||
go func() {
|
||||
// load last result from local disk
|
||||
cacheData, cacheErr := os.ReadFile(Tea.Root + "/data/" + diskSpeedDataFile)
|
||||
if cacheErr == nil {
|
||||
var cache = &DiskSpeedCache{}
|
||||
err := json.Unmarshal(cacheData, cache)
|
||||
if err == nil && cache.SpeedMB > 0 {
|
||||
DiskSpeedMB = cache.SpeedMB
|
||||
DiskSpeed = cache.Speed
|
||||
calculateDiskMaxWrites()
|
||||
}
|
||||
}
|
||||
|
||||
// initial check
|
||||
_, _, _ = CheckDiskIsFast()
|
||||
|
||||
@@ -60,6 +84,16 @@ func init() {
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// check high load
|
||||
go func() {
|
||||
var ticker = time.NewTicker(5 * time.Second)
|
||||
for range ticker.C {
|
||||
stat, _ := load.Avg()
|
||||
IsInExtremelyHighLoad = stat != nil && stat.Load1 > extremelyHighLoad1Threshold
|
||||
IsInHighLoad = stat != nil && stat.Load1 > highLoad1Threshold && !DiskIsFast()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func DiskIsFast() bool {
|
||||
@@ -67,12 +101,20 @@ func DiskIsFast() bool {
|
||||
}
|
||||
|
||||
func DiskIsExtremelyFast() bool {
|
||||
// 在开发环境下返回false,以便于测试
|
||||
if Tea.IsTesting() {
|
||||
return false
|
||||
}
|
||||
return DiskSpeed == SpeedExtremelyFast
|
||||
}
|
||||
|
||||
var countWrites int32 = 0
|
||||
|
||||
func WriteReady() bool {
|
||||
if IsInExtremelyHighLoad {
|
||||
return false
|
||||
}
|
||||
|
||||
return atomic.LoadInt32(&countWrites) < DiskMaxWrites
|
||||
}
|
||||
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
package linkedlist
|
||||
|
||||
type Item struct {
|
||||
prev *Item
|
||||
next *Item
|
||||
type Item[T any] struct {
|
||||
prev *Item[T]
|
||||
next *Item[T]
|
||||
|
||||
Value interface{}
|
||||
Value T
|
||||
}
|
||||
|
||||
func NewItem(value interface{}) *Item {
|
||||
return &Item{Value: value}
|
||||
func NewItem[T any](value T) *Item[T] {
|
||||
return &Item[T]{Value: value}
|
||||
}
|
||||
|
||||
@@ -2,25 +2,25 @@
|
||||
|
||||
package linkedlist
|
||||
|
||||
type List struct {
|
||||
head *Item
|
||||
end *Item
|
||||
type List[T any] struct {
|
||||
head *Item[T]
|
||||
end *Item[T]
|
||||
count int
|
||||
}
|
||||
|
||||
func NewList() *List {
|
||||
return &List{}
|
||||
func NewList[T any]() *List[T] {
|
||||
return &List[T]{}
|
||||
}
|
||||
|
||||
func (this *List) Head() *Item {
|
||||
func (this *List[T]) Head() *Item[T] {
|
||||
return this.head
|
||||
}
|
||||
|
||||
func (this *List) End() *Item {
|
||||
func (this *List[T]) End() *Item[T] {
|
||||
return this.end
|
||||
}
|
||||
|
||||
func (this *List) Push(item *Item) {
|
||||
func (this *List[T]) Push(item *Item[T]) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
@@ -36,7 +36,16 @@ func (this *List) Push(item *Item) {
|
||||
this.add(item)
|
||||
}
|
||||
|
||||
func (this *List) Remove(item *Item) {
|
||||
func (this *List[T]) Shift() *Item[T] {
|
||||
if this.head != nil {
|
||||
var old = this.head
|
||||
this.Remove(this.head)
|
||||
return old
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *List[T]) Remove(item *Item[T]) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
@@ -58,11 +67,11 @@ func (this *List) Remove(item *Item) {
|
||||
this.count--
|
||||
}
|
||||
|
||||
func (this *List) Len() int {
|
||||
func (this *List[T]) Len() int {
|
||||
return this.count
|
||||
}
|
||||
|
||||
func (this *List) Range(f func(item *Item) (goNext bool)) {
|
||||
func (this *List[T]) Range(f func(item *Item[T]) (goNext bool)) {
|
||||
for e := this.head; e != nil; e = e.next {
|
||||
goNext := f(e)
|
||||
if !goNext {
|
||||
@@ -71,12 +80,21 @@ func (this *List) Range(f func(item *Item) (goNext bool)) {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *List) Reset() {
|
||||
func (this *List[T]) RangeReverse(f func(item *Item[T]) (goNext bool)) {
|
||||
for e := this.end; e != nil; e = e.prev {
|
||||
goNext := f(e)
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *List[T]) Reset() {
|
||||
this.head = nil
|
||||
this.end = nil
|
||||
}
|
||||
|
||||
func (this *List) add(item *Item) {
|
||||
func (this *List[T]) add(item *Item[T]) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,7 +4,9 @@ package linkedlist_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -12,27 +14,48 @@ func TestNewList_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var list = linkedlist.NewList()
|
||||
var list = linkedlist.NewList[int]()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
var item = &linkedlist.Item{}
|
||||
var item = &linkedlist.Item[int]{}
|
||||
list.Push(item)
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
|
||||
t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
|
||||
t.Log(list.Len())
|
||||
|
||||
var count = 0
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
count++
|
||||
return true
|
||||
})
|
||||
t.Log(count)
|
||||
}
|
||||
|
||||
func TestNewList_Memory_String(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var list = linkedlist.NewList[string]()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
var item = &linkedlist.Item[string]{}
|
||||
item.Value = strconv.Itoa(i)
|
||||
list.Push(item)
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB")
|
||||
t.Log(list.Len())
|
||||
}
|
||||
|
||||
func TestList_Push(t *testing.T) {
|
||||
var list = linkedlist.NewList()
|
||||
var list = linkedlist.NewList[int]()
|
||||
list.Push(linkedlist.NewItem(1))
|
||||
list.Push(linkedlist.NewItem(2))
|
||||
|
||||
@@ -41,42 +64,84 @@ func TestList_Push(t *testing.T) {
|
||||
|
||||
var item4 = linkedlist.NewItem(4)
|
||||
list.Push(item4)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Log("=== after push3 ===")
|
||||
t.Log("=== after push 3 ===")
|
||||
list.Push(item3)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Log("=== after push4 ===")
|
||||
t.Log("=== after push 4 ===")
|
||||
list.Push(item4)
|
||||
list.Push(item3)
|
||||
list.Push(item3)
|
||||
list.Push(item3)
|
||||
list.Push(item4)
|
||||
list.Push(item4)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Log("=== after remove ===")
|
||||
t.Log("=== after remove 3 ===")
|
||||
list.Remove(item3)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func TestList_Shift(t *testing.T) {
|
||||
var list = linkedlist.NewList[int]()
|
||||
list.Push(linkedlist.NewItem(1))
|
||||
list.Push(linkedlist.NewItem(2))
|
||||
list.Push(linkedlist.NewItem(3))
|
||||
list.Push(linkedlist.NewItem(4))
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
t.Log("=== before shift " + types.String(i) + " ===")
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Logf("shift: %+v", list.Shift())
|
||||
|
||||
t.Log("=== after shift " + types.String(i) + " ===")
|
||||
list.Range(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestList_RangeReverse(t *testing.T) {
|
||||
var list = linkedlist.NewList[int]()
|
||||
list.Push(linkedlist.NewItem(1))
|
||||
list.Push(linkedlist.NewItem(2))
|
||||
|
||||
var item3 = linkedlist.NewItem(3)
|
||||
list.Push(item3)
|
||||
|
||||
list.Push(linkedlist.NewItem(4))
|
||||
|
||||
//list.Push(item3)
|
||||
//list.Remove(item3)
|
||||
list.RangeReverse(func(item *linkedlist.Item[int]) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkList_Add(b *testing.B) {
|
||||
var list = linkedlist.NewList()
|
||||
var list = linkedlist.NewList[int]()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var item = &linkedlist.Item{}
|
||||
var item = &linkedlist.Item[int]{}
|
||||
list.Push(item)
|
||||
}
|
||||
}
|
||||
|
||||
170
internal/utils/runes/runes.go
Normal file
170
internal/utils/runes/runes.go
Normal file
@@ -0,0 +1,170 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package runes
|
||||
|
||||
// ContainsAnyWordRunes 直接使用rune检查字符串是否包含任一单词
|
||||
func ContainsAnyWordRunes(s string, words [][]rune, isCaseInsensitive bool) bool {
|
||||
var allRunes = []rune(s)
|
||||
if len(allRunes) == 0 || len(words) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
var lastRune rune // last searching rune in s
|
||||
var lastIndex = -2 // -2: not started, -1: not found, >=0: rune index
|
||||
for _, wordRunes := range words {
|
||||
if len(wordRunes) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if lastIndex > -2 && lastRune == wordRunes[0] {
|
||||
if lastIndex >= 0 {
|
||||
result, _ := ContainsWordRunes(allRunes[lastIndex:], wordRunes, isCaseInsensitive)
|
||||
if result {
|
||||
return true
|
||||
}
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
result, firstIndex := ContainsWordRunes(allRunes, wordRunes, isCaseInsensitive)
|
||||
lastIndex = firstIndex
|
||||
if result {
|
||||
return true
|
||||
}
|
||||
}
|
||||
lastRune = wordRunes[0]
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ContainsAnyWord 检查字符串是否包含任一单词
|
||||
func ContainsAnyWord(s string, words []string, isCaseInsensitive bool) bool {
|
||||
var allRunes = []rune(s)
|
||||
if len(allRunes) == 0 || len(words) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
var lastRune rune // last searching rune in s
|
||||
var lastIndex = -2 // -2: not started, -1: not found, >=0: rune index
|
||||
for _, word := range words {
|
||||
var wordRunes = []rune(word)
|
||||
if len(wordRunes) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if lastIndex > -2 && lastRune == wordRunes[0] {
|
||||
if lastIndex >= 0 {
|
||||
result, _ := ContainsWordRunes(allRunes[lastIndex:], wordRunes, isCaseInsensitive)
|
||||
if result {
|
||||
return true
|
||||
}
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
result, firstIndex := ContainsWordRunes(allRunes, wordRunes, isCaseInsensitive)
|
||||
lastIndex = firstIndex
|
||||
if result {
|
||||
return true
|
||||
}
|
||||
}
|
||||
lastRune = wordRunes[0]
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ContainsAllWords 检查字符串是否包含所有单词
|
||||
func ContainsAllWords(s string, words []string, isCaseInsensitive bool) bool {
|
||||
var allRunes = []rune(s)
|
||||
if len(allRunes) == 0 || len(words) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, word := range words {
|
||||
if result, _ := ContainsWordRunes(allRunes, []rune(word), isCaseInsensitive); !result {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ContainsWordRunes 检查字符列表是否包含某个单词子字符列表
|
||||
func ContainsWordRunes(allRunes []rune, subRunes []rune, isCaseInsensitive bool) (result bool, firstIndex int) {
|
||||
firstIndex = -1
|
||||
|
||||
var l = len(subRunes)
|
||||
if l == 0 {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
var al = len(allRunes)
|
||||
|
||||
for index, r := range allRunes {
|
||||
if EqualRune(r, subRunes[0], isCaseInsensitive) && (index == 0 || !isChar(allRunes[index-1]) /**boundary check **/) {
|
||||
if firstIndex < 0 {
|
||||
firstIndex = index
|
||||
}
|
||||
|
||||
var found = true
|
||||
if l > 1 {
|
||||
for i := 1; i < l; i++ {
|
||||
var subIndex = index + i
|
||||
if subIndex > al-1 || !EqualRune(allRunes[subIndex], subRunes[i], isCaseInsensitive) {
|
||||
found = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check after charset
|
||||
if found && (al <= index+l || !isChar(allRunes[index+l]) /**boundary check **/) {
|
||||
return true, firstIndex
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false, firstIndex
|
||||
}
|
||||
|
||||
// ContainsSubRunes 检查字符列表是否包含某个子子字符列表
|
||||
// 与 ContainsWordRunes 不同,这里不需要检查边界符号
|
||||
func ContainsSubRunes(allRunes []rune, subRunes []rune, isCaseInsensitive bool) bool {
|
||||
var l = len(subRunes)
|
||||
if l == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
var al = len(allRunes)
|
||||
|
||||
for index, r := range allRunes {
|
||||
if EqualRune(r, subRunes[0], isCaseInsensitive) {
|
||||
var found = true
|
||||
if l > 1 {
|
||||
for i := 1; i < l; i++ {
|
||||
var subIndex = index + i
|
||||
if subIndex > al-1 || !EqualRune(allRunes[subIndex], subRunes[i], isCaseInsensitive) {
|
||||
found = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check after charset
|
||||
if found {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// EqualRune 判断两个rune是否相同
|
||||
func EqualRune(r1 rune, r2 rune, isCaseInsensitive bool) bool {
|
||||
const d = 'a' - 'A'
|
||||
return r1 == r2 ||
|
||||
(isCaseInsensitive && r1 >= 'a' && r1 <= 'z' && r1-r2 == d) ||
|
||||
(isCaseInsensitive && r1 >= 'A' && r1 <= 'Z' && r1-r2 == -d)
|
||||
}
|
||||
|
||||
func isChar(r rune) bool {
|
||||
return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9'
|
||||
}
|
||||
172
internal/utils/runes/runes_test.go
Normal file
172
internal/utils/runes/runes_test.go
Normal file
@@ -0,0 +1,172 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package runes_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/re"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/runes"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestContainsAllWords(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(runes.ContainsAllWords("How are you?", []string{"are", "you"}, false))
|
||||
a.IsFalse(runes.ContainsAllWords("How are you?", []string{"how", "are", "you"}, false))
|
||||
a.IsTrue(runes.ContainsAllWords("How are you?", []string{"how", "are", "you"}, true))
|
||||
}
|
||||
|
||||
func TestContainsAnyWord(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(runes.ContainsAnyWord("How are you?", []string{"are", "you"}, false))
|
||||
a.IsTrue(runes.ContainsAnyWord("How are you?", []string{"are", "you", "ok"}, false))
|
||||
a.IsFalse(runes.ContainsAnyWord("How are you?", []string{"how", "ok"}, false))
|
||||
a.IsTrue(runes.ContainsAnyWord("How are you?", []string{"how"}, true))
|
||||
a.IsTrue(runes.ContainsAnyWord("How are you?", []string{"how", "ok"}, true))
|
||||
a.IsTrue(runes.ContainsAnyWord("How-are you?", []string{"how", "ok"}, true))
|
||||
}
|
||||
|
||||
func TestContainsAnyWord_Sort(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(runes.ContainsAnyWord("How are you?", []string{"abc", "ant", "arm", "Hit", "Hi", "Pet", "pie", "are"}, false))
|
||||
}
|
||||
|
||||
func TestContainsWordRunes(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsFalse(runes.ContainsWordRunes([]rune(""), []rune("How"), true))
|
||||
a.IsFalse(runes.ContainsWordRunes([]rune("How are you?"), []rune(""), true))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you?"), []rune("How"), true))
|
||||
a.IsFalse(runes.ContainsWordRunes([]rune("How are you?"), []rune("how"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you?"), []rune("you"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you?"), []rune("are"), false))
|
||||
a.IsFalse(runes.ContainsWordRunes([]rune("How are you?"), []rune("re"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you w?"), []rune("w"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("w How are you?"), []rune("w"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are w you?"), []rune("w"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are how you?"), []rune("how"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you?"), []rune("how"), true))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you?"), []rune("ARE"), true))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you"), []rune("you"), false))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you"), []rune("YOU"), true))
|
||||
a.IsTrue(runes.ContainsWordRunes([]rune("How are you?"), []rune("YOU"), true))
|
||||
a.IsFalse(runes.ContainsWordRunes([]rune("How are you1?"), []rune("YOU"), true))
|
||||
a.IsFalse(runes.ContainsWordRunes([]rune("How are you1?"), []rune("YOU YOU YOU YOU YOU YOU YOU"), true))
|
||||
}
|
||||
|
||||
func TestContainsSubRunes(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsFalse(runes.ContainsSubRunes([]rune(""), []rune("How"), true))
|
||||
a.IsFalse(runes.ContainsSubRunes([]rune("How are you?"), []rune(""), true))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you1?"), []rune("YOU"), true))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you1?"), []rune("ow"), false))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you1?"), []rune("H"), false))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you1?"), []rune("How"), false))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you doing"), []rune("oi"), false))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you doing"), []rune("g"), false))
|
||||
a.IsTrue(runes.ContainsSubRunes([]rune("How are you doing"), []rune("ing"), false))
|
||||
a.IsFalse(runes.ContainsSubRunes([]rune("How are you doing"), []rune("int"), false))
|
||||
}
|
||||
|
||||
func TestEqualRune(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
a.IsTrue(runes.EqualRune('a', 'a', false))
|
||||
a.IsTrue(runes.EqualRune('a', 'a', true))
|
||||
a.IsFalse(runes.EqualRune('a', 'A', false))
|
||||
a.IsTrue(runes.EqualRune('a', 'A', true))
|
||||
a.IsFalse(runes.EqualRune('c', 'C', false))
|
||||
a.IsTrue(runes.EqualRune('c', 'C', true))
|
||||
a.IsTrue(runes.EqualRune('C', 'C', true))
|
||||
a.IsTrue(runes.EqualRune('C', 'c', true))
|
||||
a.IsTrue(runes.EqualRune('Z', 'z', true))
|
||||
a.IsTrue(runes.EqualRune('z', 'Z', true))
|
||||
a.IsFalse(runes.EqualRune('z', 'z'+('a'-'A'), true))
|
||||
}
|
||||
|
||||
func BenchmarkContainsWordRunes(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_, _ = runes.ContainsWordRunes([]rune("How are you"), []rune("YOU"), true)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkContainsAnyWord(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var words = strings.Split("python\npycurl\nhttp-client\nhttpclient\napachebench\nnethttp\nhttp_request\njava\nperl\nruby\nscrapy\nphp\nrust", "\n")
|
||||
sort.Strings(words)
|
||||
|
||||
var wordRunes = [][]rune{}
|
||||
for _, word := range words {
|
||||
wordRunes = append(wordRunes, []rune(word))
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_ = runes.ContainsAnyWord("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_0_0) AppleWebKit/500.00 (KHTML, like Gecko) Chrome/100.0.0.0", words, true)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkContainsAnyWordRunes(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var words = strings.Split("python\npycurl\nhttp-client\nhttpclient\napachebench\nnethttp\nhttp_request\njava\nperl\nruby\nscrapy\nphp\nrust", "\n")
|
||||
sort.Strings(words)
|
||||
|
||||
var wordRunes = [][]rune{}
|
||||
for _, word := range words {
|
||||
wordRunes = append(wordRunes, []rune(word))
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_ = runes.ContainsAnyWordRunes("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_0_0) AppleWebKit/500.00 (KHTML, like Gecko) Chrome/100.0.0.0", wordRunes, true)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
func BenchmarkContainsAnyWord_Regexp(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
var reg = regexp.MustCompile("(?i)" + strings.ReplaceAll("python\npycurl\nhttp-client\nhttpclient\napachebench\nnethttp\nhttp_request\njava\nperl\nruby\nscrapy\nphp\nrust", "\n", "|"))
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_ = reg.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_0_0) AppleWebKit/500.00 (KHTML, like Gecko) Chrome/100.0.0.0")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkContainsAnyWord_Re(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
var reg = re.MustCompile("(?i)" + strings.ReplaceAll("python\npycurl\nhttp-client\nhttpclient\napachebench\nnethttp\nhttp_request\njava\nperl\nruby\nscrapy\nphp\nrust", "\n", "|"))
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_ = reg.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_0_0) AppleWebKit/500.00 (KHTML, like Gecko) Chrome/100.0.0.0")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkContainsSubRunes(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
_ = runes.ContainsSubRunes([]rune("How are you"), []rune("YOU"), true)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -56,3 +56,16 @@ func EqualStrings(s1 []string, s2 []string) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// CutPrefix returns s without the provided leading prefix string
|
||||
// and reports whether it found the prefix.
|
||||
// If s doesn't start with prefix, CutPrefix returns s, false.
|
||||
// If prefix is the empty string, CutPrefix returns s, true.
|
||||
//
|
||||
// copy from go source
|
||||
func CutPrefix(s, prefix string) (after string, found bool) {
|
||||
if !strings.HasPrefix(s, prefix) {
|
||||
return s, false
|
||||
}
|
||||
return s[len(prefix):], true
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
)
|
||||
|
||||
var systemTotalMemory = -1
|
||||
var systemMemoryBytes uint64
|
||||
|
||||
func init() {
|
||||
if !teaconst.IsMain {
|
||||
@@ -29,7 +30,9 @@ func SystemMemoryGB() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
systemTotalMemory = int(stat.Total / (1<<30))
|
||||
systemMemoryBytes = stat.Total
|
||||
|
||||
systemTotalMemory = int(stat.Total / (1 << 30))
|
||||
if systemTotalMemory <= 0 {
|
||||
systemTotalMemory = 1
|
||||
}
|
||||
@@ -38,3 +41,8 @@ func SystemMemoryGB() int {
|
||||
|
||||
return systemTotalMemory
|
||||
}
|
||||
|
||||
// SystemMemoryBytes 系统内存总字节数
|
||||
func SystemMemoryBytes() uint64 {
|
||||
return systemMemoryBytes
|
||||
}
|
||||
|
||||
@@ -12,12 +12,7 @@ func setMaxMemory(memoryGB int) {
|
||||
if memoryGB <= 0 {
|
||||
memoryGB = 1
|
||||
}
|
||||
var maxMemoryBytes int64
|
||||
if memoryGB > 10 {
|
||||
maxMemoryBytes = int64(memoryGB-2) << 30 // 超过10G内存的允许剩余2G内存
|
||||
} else {
|
||||
maxMemoryBytes = (int64(memoryGB) << 30) * 80 / 100 // 默认 80%
|
||||
}
|
||||
|
||||
var maxMemoryBytes = (int64(memoryGB) << 30) * 80 / 100 // 默认 80%
|
||||
debug.SetMemoryLimit(maxMemoryBytes)
|
||||
}
|
||||
|
||||
@@ -8,4 +8,7 @@ func TestSystemMemoryGB(t *testing.T) {
|
||||
t.Log(SystemMemoryGB())
|
||||
t.Log(SystemMemoryGB())
|
||||
t.Log(SystemMemoryGB())
|
||||
t.Log(SystemMemoryBytes())
|
||||
t.Log(SystemMemoryBytes())
|
||||
t.Log(SystemMemoryBytes()>>30, "GB")
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ func StartMemoryStatsGC(t *testing.T) {
|
||||
}()
|
||||
}
|
||||
|
||||
func StartMemoryStats(t *testing.T) {
|
||||
func StartMemoryStats(t *testing.T, callbacks ...func()) {
|
||||
var ticker = time.NewTicker(1 * time.Second)
|
||||
go func() {
|
||||
var stat = &runtime.MemStats{}
|
||||
@@ -41,11 +41,17 @@ func StartMemoryStats(t *testing.T) {
|
||||
for range ticker.C {
|
||||
runtime.ReadMemStats(stat)
|
||||
if stat.HeapInuse == lastHeapInUse {
|
||||
return
|
||||
continue
|
||||
}
|
||||
lastHeapInUse = stat.HeapInuse
|
||||
|
||||
t.Log(timeutil.Format("H:i:s"), "HeapInuse:", fmt.Sprintf("%.2fM", float64(stat.HeapInuse)/1024/1024), "NumGC:", stat.NumGC)
|
||||
|
||||
if len(callbacks) > 0 {
|
||||
for _, callback := range callbacks {
|
||||
callback()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
package waf
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
wafutils "github.com/TeaOSLab/EdgeNode/internal/waf/utils"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -27,6 +28,8 @@ type CaptchaAction struct {
|
||||
|
||||
CountLetters int8 `yaml:"countLetters" json:"countLetters"`
|
||||
|
||||
CaptchaType firewallconfigs.CaptchaType `yaml:"captchaType" json:"captchaType"`
|
||||
|
||||
UIIsOn bool `yaml:"uiIsOn" json:"uiIsOn"` // 是否使用自定义UI
|
||||
UITitle string `yaml:"uiTitle" json:"uiTitle"` // 消息标题
|
||||
UIPrompt string `yaml:"uiPrompt" json:"uiPrompt"` // 消息提示
|
||||
@@ -36,6 +39,24 @@ type CaptchaAction struct {
|
||||
UIFooter string `yaml:"uiFooter" json:"uiFooter"` // 页脚
|
||||
UIBody string `yaml:"uiBody" json:"uiBody"` // 内容轮廓
|
||||
|
||||
OneClickUIIsOn bool `yaml:"oneClickUIIsOn" json:"oneClickUIIsOn"` // 是否使用自定义UI
|
||||
OneClickUITitle string `yaml:"oneClickUITitle" json:"oneClickUITitle"` // 消息标题
|
||||
OneClickUIPrompt string `yaml:"oneClickUIPrompt" json:"oneClickUIPrompt"` // 消息提示
|
||||
OneClickUIShowRequestId bool `yaml:"oneClickUIShowRequestId" json:"oneClickUIShowRequestId"` // 是否显示请求ID
|
||||
OneClickUICss string `yaml:"oneClickUICss" json:"oneClickUICss"` // CSS样式
|
||||
OneClickUIFooter string `yaml:"oneClickUIFooter" json:"oneClickUIFooter"` // 页脚
|
||||
OneClickUIBody string `yaml:"oneClickUIBody" json:"oneClickUIBody"` // 内容轮廓
|
||||
|
||||
SlideUIIsOn bool `yaml:"sliceUIIsOn" json:"sliceUIIsOn"` // 是否使用自定义UI
|
||||
SlideUITitle string `yaml:"slideUITitle" json:"slideUITitle"` // 消息标题
|
||||
SlideUIPrompt string `yaml:"slideUIPrompt" json:"slideUIPrompt"` // 消息提示
|
||||
SlideUIShowRequestId bool `yaml:"SlideUIShowRequestId" json:"SlideUIShowRequestId"` // 是否显示请求ID
|
||||
SlideUICss string `yaml:"slideUICss" json:"slideUICss"` // CSS样式
|
||||
SlideUIFooter string `yaml:"slideUIFooter" json:"slideUIFooter"` // 页脚
|
||||
SlideUIBody string `yaml:"slideUIBody" json:"slideUIBody"` // 内容轮廓
|
||||
|
||||
GeeTestConfig *firewallconfigs.GeeTestConfig `yaml:"geeTestConfig" json:"geeTestConfig"` // 极验设置 MUST be struct
|
||||
|
||||
Lang string `yaml:"lang" json:"lang"` // 语言,zh-CN, en-US ...
|
||||
AddToWhiteList bool `yaml:"addToWhiteList" json:"addToWhiteList"` // 是否加入到白名单
|
||||
Scope string `yaml:"scope" json:"scope"`
|
||||
@@ -81,6 +102,10 @@ func (this *CaptchaAction) Init(waf *WAF) error {
|
||||
if len(this.Lang) == 0 {
|
||||
this.Lang = waf.DefaultCaptchaAction.Lang
|
||||
}
|
||||
|
||||
if len(this.CaptchaType) == 0 {
|
||||
this.CaptchaType = waf.DefaultCaptchaAction.CaptchaType
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -100,7 +125,7 @@ func (this *CaptchaAction) WillChange() bool {
|
||||
|
||||
func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// 是否在白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, req.WAFServerId(), req.WAFRemoteIP()) {
|
||||
if SharedIPWhiteList.Contains(wafutils.ComposeIPType(set.Id, req), this.Scope, req.WAFServerId(), req.WAFRemoteIP()) {
|
||||
return true, false
|
||||
}
|
||||
|
||||
@@ -134,6 +159,7 @@ func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
// 占用一次失败次数
|
||||
CaptchaIncreaseFails(req, this, waf.Id, group.Id, set.Id, CaptchaPageCodeInit)
|
||||
|
||||
req.DisableStat()
|
||||
req.ProcessResponseHeaders(writer.Header(), http.StatusTemporaryRedirect)
|
||||
http.Redirect(writer, req.WAFRaw(), CaptchaPath+"?info="+url.QueryEscape(info), http.StatusTemporaryRedirect)
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
|
||||
return true, false
|
||||
}
|
||||
|
||||
request.DisableStat()
|
||||
request.ProcessResponseHeaders(writer.Header(), http.StatusFound)
|
||||
http.Redirect(writer, request.WAFRaw(), Get302Path+"?info="+url.QueryEscape(info), http.StatusFound)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/counters"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
@@ -118,9 +119,9 @@ func (this *JSCookieAction) increaseFails(req requests.Request, policyId int64,
|
||||
failBlockTimeout = 1800 // 默认1800s
|
||||
}
|
||||
|
||||
var key = "JS_COOKIE:FAILS:" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + req.WAFRaw().URL.String()
|
||||
var key = "WAF:JS_COOKIE:FAILS:" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + req.WAFRaw().URL.String()
|
||||
|
||||
var countFails = SharedCounter.IncreaseKey(key, 300)
|
||||
var countFails = counters.SharedCounter.IncreaseKey(key, 300)
|
||||
if int(countFails) >= maxFails {
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, true, groupId, setId, "JS_COOKIE验证连续失败超过"+types.String(maxFails)+"次")
|
||||
return false
|
||||
|
||||
@@ -36,10 +36,30 @@ func (this *PageAction) WillChange() bool {
|
||||
|
||||
// Perform the action
|
||||
func (this *PageAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
if writer == nil {
|
||||
return
|
||||
}
|
||||
|
||||
request.ProcessResponseHeaders(writer.Header(), this.Status)
|
||||
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
writer.WriteHeader(this.Status)
|
||||
_, _ = writer.Write([]byte(request.Format(this.Body)))
|
||||
|
||||
var body = this.Body
|
||||
if len(body) == 0 {
|
||||
body = `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<title>403 Forbidden</title>
|
||||
<style>
|
||||
address { line-height: 1.8; }
|
||||
</style>
|
||||
<body>
|
||||
<h1>403 Forbidden By WAF</h1>
|
||||
<address>Connection: ${remoteAddr} (Client) -> ${serverAddr} (Server)</address>
|
||||
<address>Request ID: ${requestId}</address>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
_, _ = writer.Write([]byte(request.Format(body)))
|
||||
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -92,6 +92,7 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
Value: info,
|
||||
})
|
||||
|
||||
request.DisableStat()
|
||||
request.ProcessResponseHeaders(writer.Header(), http.StatusTemporaryRedirect)
|
||||
http.Redirect(writer, request.WAFRaw(), request.WAFRaw().URL.String(), http.StatusTemporaryRedirect)
|
||||
|
||||
|
||||
@@ -133,8 +133,13 @@ func (this *RecordIPAction) WillChange() bool {
|
||||
}
|
||||
|
||||
func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
var ipListId = this.IPListId
|
||||
if ipListId <= 0 {
|
||||
ipListId = firewallconfigs.GlobalListId
|
||||
}
|
||||
|
||||
// 是否已删除
|
||||
var ipListIsAvailable = this.IPListId > 0 && !this.IPListIsDeleted && !ExistDeletedIPList(this.IPListId)
|
||||
var ipListIsAvailable = (ipListId == firewallconfigs.GlobalListId) || (ipListId > 0 && !this.IPListIsDeleted && !ExistDeletedIPList(ipListId))
|
||||
|
||||
// 是否在本地白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
|
||||
@@ -167,7 +172,7 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
|
||||
}
|
||||
|
||||
// 上报
|
||||
if this.IPListId > 0 && ipListIsAvailable {
|
||||
if ipListId > 0 && ipListIsAvailable {
|
||||
var serverId int64
|
||||
if this.Scope == firewallconfigs.FirewallScopeService {
|
||||
serverId = request.WAFServerId()
|
||||
@@ -181,7 +186,7 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
|
||||
select {
|
||||
case recordIPTaskChan <- &recordIPTask{
|
||||
ip: request.WAFRemoteIP(),
|
||||
listId: this.IPListId,
|
||||
listId: ipListId,
|
||||
expiresAt: realExpiresAt,
|
||||
level: this.Level,
|
||||
serverId: serverId,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package waf
|
||||
package waf_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
@@ -11,22 +12,22 @@ import (
|
||||
func TestFindActionInstance(t *testing.T) {
|
||||
a := assert.NewAssertion(t)
|
||||
|
||||
t.Logf("ActionBlock: %p", FindActionInstance(ActionBlock, nil))
|
||||
t.Logf("ActionBlock: %p", FindActionInstance(ActionBlock, nil))
|
||||
t.Logf("ActionGoGroup: %p", FindActionInstance(ActionGoGroup, nil))
|
||||
t.Logf("ActionGoGroup: %p", FindActionInstance(ActionGoGroup, nil))
|
||||
t.Logf("ActionGoSet: %p", FindActionInstance(ActionGoSet, nil))
|
||||
t.Logf("ActionGoSet: %p", FindActionInstance(ActionGoSet, nil))
|
||||
t.Logf("ActionGoSet: %#v", FindActionInstance(ActionGoSet, maps.Map{"groupId": "a", "setId": "b"}))
|
||||
t.Logf("ActionBlock: %p", waf.FindActionInstance(waf.ActionBlock, nil))
|
||||
t.Logf("ActionBlock: %p", waf.FindActionInstance(waf.ActionBlock, nil))
|
||||
t.Logf("ActionGoGroup: %p", waf.FindActionInstance(waf.ActionGoGroup, nil))
|
||||
t.Logf("ActionGoGroup: %p", waf.FindActionInstance(waf.ActionGoGroup, nil))
|
||||
t.Logf("ActionGoSet: %p", waf.FindActionInstance(waf.ActionGoSet, nil))
|
||||
t.Logf("ActionGoSet: %p", waf.FindActionInstance(waf.ActionGoSet, nil))
|
||||
t.Logf("ActionGoSet: %#v", waf.FindActionInstance(waf.ActionGoSet, maps.Map{"groupId": "a", "setId": "b"}))
|
||||
|
||||
a.IsTrue(FindActionInstance(ActionGoSet, nil) != FindActionInstance(ActionGoSet, nil))
|
||||
a.IsTrue(waf.FindActionInstance(waf.ActionGoSet, nil) != waf.FindActionInstance(waf.ActionGoSet, nil))
|
||||
}
|
||||
|
||||
func TestFindActionInstance_Options(t *testing.T) {
|
||||
//t.Logf("%p", FindActionInstance(ActionBlock, maps.Map{}))
|
||||
//t.Logf("%p", FindActionInstance(ActionBlock, maps.Map{}))
|
||||
//logs.PrintAsJSON(FindActionInstance(ActionBlock, maps.Map{}), t)
|
||||
logs.PrintAsJSON(FindActionInstance(ActionBlock, maps.Map{
|
||||
logs.PrintAsJSON(waf.FindActionInstance(waf.ActionBlock, maps.Map{
|
||||
"timeout": 3600,
|
||||
}), t)
|
||||
}
|
||||
@@ -34,6 +35,6 @@ func TestFindActionInstance_Options(t *testing.T) {
|
||||
func BenchmarkFindActionInstance(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
for i := 0; i < b.N; i++ {
|
||||
FindActionInstance(ActionGoSet, nil)
|
||||
waf.FindActionInstance(waf.ActionGoSet, nil)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ package waf
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/counters"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"time"
|
||||
@@ -15,6 +15,7 @@ type CaptchaPageCode = string
|
||||
const (
|
||||
CaptchaPageCodeInit CaptchaPageCode = "init"
|
||||
CaptchaPageCodeShow CaptchaPageCode = "show"
|
||||
CaptchaPageCodeImage CaptchaPageCode = "image"
|
||||
CaptchaPageCodeSubmit CaptchaPageCode = "submit"
|
||||
)
|
||||
|
||||
@@ -26,7 +27,7 @@ func CaptchaIncreaseFails(req requests.Request, actionConfig *CaptchaAction, pol
|
||||
if maxFails <= 3 {
|
||||
maxFails = 3 // 不能小于3,防止意外刷新出现
|
||||
}
|
||||
var countFails = SharedCounter.IncreaseKey(CaptchaCacheKey(req, pageCode), 300)
|
||||
var countFails = counters.SharedCounter.IncreaseKey(CaptchaCacheKey(req, pageCode), 300)
|
||||
if int(countFails) >= maxFails {
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, true, groupId, setId, "CAPTCHA验证连续失败超过"+types.String(maxFails)+"次")
|
||||
return false
|
||||
@@ -37,21 +38,13 @@ func CaptchaIncreaseFails(req requests.Request, actionConfig *CaptchaAction, pol
|
||||
|
||||
// CaptchaDeleteCacheKey 清除计数
|
||||
func CaptchaDeleteCacheKey(req requests.Request) {
|
||||
SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeInit))
|
||||
SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeShow))
|
||||
SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeSubmit))
|
||||
counters.SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeInit))
|
||||
counters.SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeShow))
|
||||
counters.SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeImage))
|
||||
counters.SharedCounter.ResetKey(CaptchaCacheKey(req, CaptchaPageCodeSubmit))
|
||||
}
|
||||
|
||||
// CaptchaCacheKey 获取Captcha缓存Key
|
||||
func CaptchaCacheKey(req requests.Request, pageCode CaptchaPageCode) string {
|
||||
var requestPath = req.WAFRaw().URL.Path
|
||||
|
||||
if req.WAFRaw().URL.Path == CaptchaPath {
|
||||
m, err := utils.SimpleDecryptMap(req.WAFRaw().URL.Query().Get("info"))
|
||||
if err == nil && m != nil {
|
||||
requestPath = m.GetString("url")
|
||||
}
|
||||
}
|
||||
|
||||
return "CAPTCHA:FAILS:" + pageCode + ":" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + requestPath
|
||||
return "WAF:CAPTCHA:FAILS:" + pageCode + ":" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId())
|
||||
}
|
||||
|
||||
71
internal/waf/captcha_generator.go
Normal file
71
internal/waf/captcha_generator.go
Normal file
@@ -0,0 +1,71 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package waf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/dchest/captcha"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CaptchaGenerator captcha generator
|
||||
type CaptchaGenerator struct {
|
||||
store captcha.Store
|
||||
}
|
||||
|
||||
func NewCaptchaGenerator() *CaptchaGenerator {
|
||||
return &CaptchaGenerator{
|
||||
store: captcha.NewMemoryStore(100_000, 5*time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
// NewCaptcha create new captcha
|
||||
func (this *CaptchaGenerator) NewCaptcha(length int) (captchaId string) {
|
||||
captchaId = rands.HexString(16)
|
||||
|
||||
if length <= 0 || length > 20 {
|
||||
length = 4
|
||||
}
|
||||
|
||||
this.store.Set(captchaId, captcha.RandomDigits(length))
|
||||
return
|
||||
}
|
||||
|
||||
// WriteImage write image to front writer
|
||||
func (this *CaptchaGenerator) WriteImage(w io.Writer, id string, width, height int) error {
|
||||
var d = this.store.Get(id, false)
|
||||
if d == nil {
|
||||
return captcha.ErrNotFound
|
||||
}
|
||||
_, err := captcha.NewImage(id, d, width, height).WriteTo(w)
|
||||
return err
|
||||
}
|
||||
|
||||
// Verify user input
|
||||
func (this *CaptchaGenerator) Verify(id string, digits string) bool {
|
||||
var countDigits = len(digits)
|
||||
if countDigits == 0 {
|
||||
return false
|
||||
}
|
||||
var value = this.store.Get(id, true)
|
||||
if len(value) != countDigits {
|
||||
return false
|
||||
}
|
||||
|
||||
var nb = make([]byte, countDigits)
|
||||
for i := 0; i < countDigits; i++ {
|
||||
var d = digits[i]
|
||||
if d >= '0' && d <= '9' {
|
||||
nb[i] = d - '0'
|
||||
}
|
||||
}
|
||||
|
||||
return bytes.Equal(nb, value)
|
||||
}
|
||||
|
||||
// Get captcha data
|
||||
func (this *CaptchaGenerator) Get(id string) []byte {
|
||||
return this.store.Get(id, false)
|
||||
}
|
||||
87
internal/waf/captcha_generator_test.go
Normal file
87
internal/waf/captcha_generator_test.go
Normal file
@@ -0,0 +1,87 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package waf_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCaptchaGenerator_NewCaptcha(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var generator = waf.NewCaptchaGenerator()
|
||||
var captchaId = generator.NewCaptcha(6)
|
||||
t.Log("captchaId:", captchaId)
|
||||
|
||||
var digits = generator.Get(captchaId)
|
||||
var s []string
|
||||
for _, digit := range digits {
|
||||
s = append(s, types.String(digit))
|
||||
}
|
||||
t.Log(strings.Join(s, " "))
|
||||
|
||||
a.IsTrue(generator.Verify(captchaId, strings.Join(s, "")))
|
||||
a.IsFalse(generator.Verify(captchaId, strings.Join(s, "")))
|
||||
}
|
||||
|
||||
func TestCaptchaGenerator_NewCaptcha_UTF8(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var generator = waf.NewCaptchaGenerator()
|
||||
var captchaId = generator.NewCaptcha(6)
|
||||
t.Log("captchaId:", captchaId)
|
||||
|
||||
var digits = generator.Get(captchaId)
|
||||
var s []string
|
||||
for _, digit := range digits {
|
||||
s = append(s, types.String(digit))
|
||||
}
|
||||
t.Log(strings.Join(s, " "))
|
||||
|
||||
a.IsFalse(generator.Verify(captchaId, "中文真的很长"))
|
||||
}
|
||||
|
||||
func TestCaptchaGenerator_NewCaptcha_Memory(t *testing.T) {
|
||||
runtime.GC()
|
||||
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var generator = waf.NewCaptchaGenerator()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
generator.NewCaptcha(6)
|
||||
}
|
||||
|
||||
if testutils.IsSingleTesting() {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log((stat2.HeapInuse-stat1.HeapInuse)>>10, "KiB")
|
||||
|
||||
_ = generator
|
||||
}
|
||||
|
||||
func BenchmarkNewCaptchaGenerator(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
var generator = waf.NewCaptchaGenerator()
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
generator.NewCaptcha(6)
|
||||
}
|
||||
})
|
||||
}
|
||||
70
internal/waf/captcha_test.go
Normal file
70
internal/waf/captcha_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package waf_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/dchest/captcha"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCaptchaMemory(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
return
|
||||
}
|
||||
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var count = 5_000
|
||||
var before = time.Now()
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
var id = captcha.NewLen(6)
|
||||
var writer = &bytes.Buffer{}
|
||||
err := captcha.WriteImage(writer, id, 200, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
captcha.VerifyString(id, "abc")
|
||||
}
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
t.Log((stat2.HeapInuse-stat1.HeapInuse)>>20, "MB", fmt.Sprintf("%.0f QPS", float64(count)/time.Since(before).Seconds()))
|
||||
}
|
||||
|
||||
func BenchmarkCaptcha_VerifyCode_100_50(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var id = captcha.NewLen(6)
|
||||
var writer = &bytes.Buffer{}
|
||||
err := captcha.WriteImage(writer, id, 100, 50)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkCaptcha_VerifyCode_200_100(b *testing.B) {
|
||||
runtime.GOMAXPROCS(4)
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
var id = captcha.NewLen(6)
|
||||
var writer = &bytes.Buffer{}
|
||||
err := captcha.WriteImage(writer, id, 200, 100)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_ = id
|
||||
}
|
||||
})
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user