Compare commits

..

69 Commits

Author SHA1 Message Date
刘祥超
83f581f7f0 清理缓存时也清理HEAD缓存 2022-02-22 21:52:28 +08:00
刘祥超
03ce082926 支持对GET/POST之外的请求方法独立缓存 2022-02-22 21:43:47 +08:00
刘祥超
ebafe458ff 支持缓存压缩后的内容 2022-02-22 19:29:27 +08:00
刘祥超
ff30b8d15c 修改版本为v0.4.3 2022-02-21 18:31:17 +08:00
刘祥超
51dd778ca7 更改为v0.4.2 2022-02-21 17:52:02 +08:00
刘祥超
b5f706686c 修复热点数据从文件系统转移到内存时可能不完整的Bug/实现部分Partial Content功能 2022-02-21 17:33:58 +08:00
刘祥超
b67c2ec39c 缓存关闭X-Cache显示时从Header中删除X-Cache 2022-02-21 16:55:25 +08:00
刘祥超
d40bc4e72b URL跳转可以设置是否保留参数 2022-02-20 09:17:50 +08:00
刘祥超
94d0fc7e88 当压缩格式不在Accept-Encoding中时自动解压 2022-02-18 11:05:09 +08:00
刘祥超
ceaeba7089 修复文件句柄缓存可能重复加入的Bug 2022-02-17 17:38:56 +08:00
刘祥超
a1e868bf29 读取缓存错误更详细 2022-02-17 17:24:35 +08:00
刘祥超
e60af85819 修复从缓存文件中读取压缩内容时可能失败的Bug 2022-02-17 16:56:13 +08:00
刘祥超
7bd24fcc81 检查是否压缩的时候,如果content-type为空,则默认为text/html 2022-02-15 18:31:37 +08:00
刘祥超
4331223916 优化代码 2022-02-15 16:44:39 +08:00
刘祥超
f50113517a 重构对HTTP请求的处理方法:缓存、压缩、WebP、限速 2022-02-15 14:55:49 +08:00
刘祥超
6d6e25f298 WAF规则提示错误时增加分组ID、规则集ID、规则描述 2022-01-29 21:43:42 +08:00
刘祥超
69c89fda48 支持单个服务更新配置 2022-01-19 22:16:46 +08:00
刘祥超
9a56671457 修改版本为v0.4.1 2022-01-17 10:53:23 +08:00
刘祥超
7dde0deb25 TCP源站也支持证书 2022-01-16 19:58:07 +08:00
刘祥超
b4647b1baa 优化验证码在窄屏上的展示 2022-01-16 16:57:25 +08:00
刘祥超
952e3ca572 CAPTCHA增加多个选项 2022-01-16 16:54:13 +08:00
刘祥超
238973a5e2 删除缓存数据库版本切换时的错误提示 2022-01-14 11:48:39 +08:00
刘祥超
9b6ab2fa8b 优化代码 2022-01-14 11:21:28 +08:00
刘祥超
9591004b70 优化代码 2022-01-13 15:49:42 +08:00
刘祥超
00cd86a8b3 优化open file cache,现在能缓存header 2022-01-13 15:18:49 +08:00
刘祥超
2a1cc63989 优化代码 2022-01-13 11:46:42 +08:00
刘祥超
8177768cf6 优化代码 2022-01-13 11:45:51 +08:00
刘祥超
14d156d42d 改进SYN Flood检测 2022-01-13 11:36:05 +08:00
刘祥超
63992bb2a0 优化代码 2022-01-12 21:41:05 +08:00
刘祥超
d02f9f9a0e 实现open file cache 2022-01-12 21:09:00 +08:00
刘祥超
91fab59a18 优化代码 2022-01-12 20:31:04 +08:00
刘祥超
76c82b431a 优化代码 2022-01-11 17:17:58 +08:00
刘祥超
2f6414fc55 优化代码 2022-01-11 16:02:41 +08:00
刘祥超
e23f4aaee2 优化代码 2022-01-11 09:25:34 +08:00
刘祥超
443660ac38 实现自动SYN Flood防护 2022-01-10 19:54:10 +08:00
刘祥超
488430bbef 优化代码 2022-01-10 15:38:53 +08:00
刘祥超
344de90bff 部分请求增加User-Agent 2022-01-10 10:02:15 +08:00
刘祥超
2f02827cb7 优化编译脚本 2022-01-09 20:12:59 +08:00
刘祥超
03e774cc44 自动使用本地防火墙/增加edge-node [ip.drop|ip.reject|ip.remove]等命令 2022-01-09 17:07:37 +08:00
刘祥超
ff2826ab47 优化一处错误提示 2022-01-09 15:32:02 +08:00
刘祥超
ecaa45db34 优化代码 2022-01-09 10:53:21 +08:00
刘祥超
b6cc826a54 优化正则表达式/修复一些测试用例 2022-01-08 12:20:18 +08:00
刘祥超
b8d7e3f5b4 提升WAF正则表达式性能(提升20%以上) 2022-01-08 11:45:14 +08:00
刘祥超
390be7f6c6 增加${browser.xxx}相关变量 2022-01-06 17:05:04 +08:00
刘祥超
ac4e240912 国家、省份数据不再每个小时更新一次;WAF增加国家/地区、省份、城市、ISP等参数 2022-01-06 16:27:39 +08:00
刘祥超
be7267211b 统计数据上传时如果遇到invalid utf-8,则自动过滤非法字符/统计数据上传失败时,仍然丢弃已有的统计数据,防止数据堆积 2022-01-05 16:05:58 +08:00
刘祥超
88fa75acb5 优化代码 2022-01-03 21:50:51 +08:00
刘祥超
d62fccf0a4 如果源站返回的内容长度为0,则不再尝试读取数据 2022-01-03 18:10:02 +08:00
刘祥超
258ffef0c2 尝试自动在firewalld中开放端口 2022-01-03 16:27:34 +08:00
刘祥超
a41f834192 可以打印服务相关日志信息 2022-01-03 15:53:59 +08:00
刘祥超
00500cb6a3 优化代码 2022-01-02 22:45:37 +08:00
刘祥超
32a3400138 优化代码 2022-01-01 22:02:46 +08:00
刘祥超
5ae4ef665e 优化UserAgent解析 2022-01-01 21:47:59 +08:00
刘祥超
336db828ad 优化代码 2022-01-01 20:15:39 +08:00
刘祥超
a1212804bb 增加edge-node gc命令 2022-01-01 17:18:34 +08:00
刘祥超
763ab4ac98 优化代码 2021-12-31 19:51:56 +08:00
刘祥超
4ec6ae4301 优化文字提示 2021-12-31 19:46:33 +08:00
刘祥超
4f292c5003 如果没有设置节点CPU线程数,则默认为4倍的CPU线程数 2021-12-31 19:45:54 +08:00
刘祥超
a00325f41a 修改版本为0.4.0 2021-12-31 15:19:20 +08:00
刘祥超
fc4490b782 修改版本号0.3.8 2021-12-31 11:36:11 +08:00
刘祥超
7f3f7e21b8 将请求的一些方法改为可exported,方便以后扩展 2021-12-30 11:19:11 +08:00
刘祥超
2525cdc061 根据Accept-Encoding决定是否解压响应内容 2021-12-29 10:57:15 +08:00
刘祥超
4ffc619aad 将获取系统内存函数放入到utils中 2021-12-29 10:53:59 +08:00
刘祥超
f930705fd7 删除不需要的文件 2021-12-24 15:00:49 +08:00
刘祥超
56961c1476 修复测试用例 2021-12-23 14:36:52 +08:00
刘祥超
28b61d493f 优化代码 2021-12-22 16:43:16 +08:00
刘祥超
5cb5ddf2c1 修复并发下,写缓存文件可能冲突的问题 2021-12-21 08:03:09 +08:00
刘祥超
fd0bc37ec7 修复并发下,写缓存文件可能冲突的问题 2021-12-21 00:27:32 +08:00
刘祥超
73666bea7f 修改版本号为0.4.0 2021-12-20 20:01:07 +08:00
162 changed files with 5727 additions and 2319 deletions

View File

@@ -6,6 +6,9 @@ function build() {
VERSION=$(lookup-version $ROOT/../internal/const/const.go)
DIST=$ROOT/"../dist/${NAME}"
MUSL_DIR="/usr/local/opt/musl-cross/bin"
GCC_X86_64_DIR="/usr/local/Cellar/x86_64-unknown-linux-gnu/10.3.0/bin"
GCC_ARM64_DIR="/usr/local/Cellar/aarch64-unknown-linux-gnu/10.3.0/bin"
OS=${1}
ARCH=${2}
TAG=${3}
@@ -56,19 +59,39 @@ function build() {
CC_PATH=""
CXX_PATH=""
BUILD_TAG=$TAG
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
# /usr/local/opt/musl-cross/bin/
if [ "${ARCH}" == "amd64" ]; then
CC_PATH="x86_64-linux-musl-gcc"
CXX_PATH="x86_64-linux-musl-g++"
# build with script support
if [ -d $GCC_X86_64_DIR ]; then
MUSL_DIR=$GCC_X86_64_DIR
CC_PATH="x86_64-unknown-linux-gnu-gcc"
CXX_PATH="x86_64-unknown-linux-gnu-g++"
if [ "$TAG" = "plus" ]; then
BUILD_TAG="plus,script"
fi
else
CC_PATH="x86_64-linux-musl-gcc"
CXX_PATH="x86_64-linux-musl-g++"
fi
fi
if [ "${ARCH}" == "386" ]; then
CC_PATH="i486-linux-musl-gcc"
CXX_PATH="i486-linux-musl-g++"
fi
if [ "${ARCH}" == "arm64" ]; then
CC_PATH="aarch64-linux-musl-gcc"
CXX_PATH="aarch64-linux-musl-g++"
# build with script support
if [ -d $GCC_ARM64_DIR ]; then
MUSL_DIR=$GCC_ARM64_DIR
CC_PATH="aarch64-unknown-linux-gnu-gcc"
CXX_PATH="aarch64-unknown-linux-gnu-g++"
if [ "$TAG" = "plus" ]; then
BUILD_TAG="plus,script"
fi
else
CC_PATH="aarch64-linux-musl-gcc"
CXX_PATH="aarch64-linux-musl-g++"
fi
fi
if [ "${ARCH}" == "arm" ]; then
CC_PATH="arm-linux-musleabi-gcc"
@@ -84,7 +107,7 @@ function build() {
fi
fi
if [ ! -z $CC_PATH ]; then
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $BUILD_TAG -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
else
env GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
fi

View File

@@ -8,7 +8,10 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/nodes"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"github.com/iwind/gosock/pkg/gosock"
"net"
"net/http"
_ "net/http/pprof"
"os"
@@ -117,6 +120,122 @@ func main() {
}
}
})
app.On("gc", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
_, err := sock.Send(&gosock.Command{Code: "gc"})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
} else {
fmt.Println("ok")
}
})
app.On("ip.drop", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS]")
return
}
var ip = args[0]
if len(net.ParseIP(ip)) == 0 {
fmt.Println("IP '" + ip + "' is invalid")
return
}
var timeoutSeconds = 0
var options = app.ParseOptions(args[1:])
timeout, ok := options["timeout"]
if ok {
timeoutSeconds = types.Int(timeout[0])
}
fmt.Println("drop ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{
Code: "dropIP",
Params: map[string]interface{}{
"ip": ip,
"timeoutSeconds": timeoutSeconds,
},
})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
} else {
var errString = maps.NewMap(reply.Params).GetString("error")
if len(errString) > 0 {
fmt.Println("[ERROR]" + errString)
} else {
fmt.Println("ok")
}
}
})
app.On("ip.reject", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.reject IP [--timeout=SECONDS]")
return
}
var ip = args[0]
if len(net.ParseIP(ip)) == 0 {
fmt.Println("IP '" + ip + "' is invalid")
return
}
var timeoutSeconds = 0
var options = app.ParseOptions(args[1:])
timeout, ok := options["timeout"]
if ok {
timeoutSeconds = types.Int(timeout[0])
}
fmt.Println("reject ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{
Code: "rejectIP",
Params: map[string]interface{}{
"ip": ip,
"timeoutSeconds": timeoutSeconds,
},
})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
} else {
var errString = maps.NewMap(reply.Params).GetString("error")
if len(errString) > 0 {
fmt.Println("[ERROR]" + errString)
} else {
fmt.Println("ok")
}
}
})
app.On("ip.remove", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.remove IP")
return
}
var ip = args[0]
if len(net.ParseIP(ip)) == 0 {
fmt.Println("IP '" + ip + "' is invalid")
return
}
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{
Code: "removeIP",
Params: map[string]interface{}{
"ip": ip,
},
})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
} else {
var errString = maps.NewMap(reply.Params).GetString("error")
if len(errString) > 0 {
fmt.Println("[ERROR]" + errString)
} else {
fmt.Println("ok")
}
}
})
app.Run(func() {
node := nodes.NewNode()
node.Start()

13
go.mod
View File

@@ -2,7 +2,9 @@ module github.com/TeaOSLab/EdgeNode
go 1.15
replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
replace (
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
)
require (
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
@@ -11,25 +13,28 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/chai2010/webp v1.1.0 // indirect
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/golang/protobuf v1.5.2
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-sqlite3 v1.14.9
github.com/miekg/dns v1.1.43
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/tklauser/go-sysconf v0.3.6 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
golang.org/x/text v0.3.7
golang.org/x/tools v0.1.3 // indirect
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/grpc v1.43.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
)

57
go.sum
View File

@@ -6,8 +6,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -36,28 +34,22 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49 h1:CtSi0QlA2Hy+nOh8JAZoiEBLW5pliAiKJ3l1Iq1472I=
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
@@ -89,7 +81,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060 h1:qdLtK4PDXxk2vMKkTWl5Fl9xqYuRCukzWAgJbLHdfOo=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24 h1:1cGulkD2SNJJRok5OKwyhP/Ddm+PgSWKOupn0cR36/A=
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
@@ -100,8 +91,6 @@ github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -111,22 +100,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo=
github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -145,8 +124,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
@@ -154,14 +131,15 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -171,18 +149,15 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -196,8 +171,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 h1:kmreh1vGI63l2FxOAYS3Yv6ATsi7lSTuwNSVbGfJV9I=
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -214,6 +187,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -226,16 +200,16 @@ golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d h1:1oIt9o40TWWI9FUaveVpUvBe13FNqBNVXy3ue2fcfkw=
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -244,11 +218,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
@@ -260,8 +230,6 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0=
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -272,8 +240,6 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
@@ -287,7 +253,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

View File

@@ -11,6 +11,7 @@ import (
"os/exec"
"runtime"
"strconv"
"strings"
"syscall"
"time"
)
@@ -245,3 +246,19 @@ func (this *AppCmd) getPID() int {
}
return maps.NewMap(reply.Params).GetInt("pid")
}
// ParseOptions 分析参数中的选项
func (this *AppCmd) ParseOptions(args []string) map[string][]string {
var result = map[string][]string{}
for _, arg := range args {
var pieces = strings.SplitN(arg, "=", 2)
var key = strings.TrimLeft(pieces[0], "- ")
key = strings.TrimSpace(key)
var value = ""
if len(pieces) == 2 {
value = strings.TrimSpace(pieces[1])
}
result[key] = append(result[key], value)
}
return result
}

View File

@@ -206,6 +206,7 @@ func (this *FileList) Add(hash string, item *Item) error {
return err
}
this.memoryCache.Write(hash, 1, item.ExpiredAt)
atomic.AddInt64(&this.total, 1)
if this.onAdd != nil {
@@ -258,9 +259,10 @@ func (this *FileList) CleanPrefix(prefix string) error {
}()
var count = int64(10000)
var staleLife = 600 // TODO 需要可以设置
var staleLife = 600 // TODO 需要可以设置
var unixTime = utils.UnixTime() // 只删除当前的,不删除新的
for {
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, utils.UnixTime()+int64(staleLife), utils.UnixTime(), prefix)
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
if err != nil {
return err
}
@@ -558,9 +560,7 @@ ON "` + this.itemsTableName + `" (
// v2 => v3
remotelogs.Println("CACHE", "transferring old data from v2 to v3 ...")
result, err := db.Exec(`INSERT INTO "` + this.itemsTableName + `" ("id", "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "createdAt", "host", "serverId", "staleAt") SELECT "id", "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "createdAt", "host", "serverId", "expiredAt"+600 FROM cacheItems_v2`)
if err != nil {
remotelogs.Println("CACHE", "transfer old data from v2 to v3 failed: "+err.Error())
} else {
if err == nil {
count, _ := result.RowsAffected()
remotelogs.Println("CACHE", "transfer old data from v2 to v3 finished, "+types.String(count)+" rows transferred")
}

View File

@@ -85,7 +85,7 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
for _, policy := range this.policyMap {
storage, ok := this.storageMap[policy.Id]
if !ok {
storage := this.NewStorageWithPolicy(policy)
storage = this.NewStorageWithPolicy(policy)
if storage == nil {
remotelogs.Error("CACHE", "can not find storage type '"+policy.Type+"'")
continue
@@ -106,7 +106,7 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
delete(this.storageMap, policy.Id)
// 启动新的
storage := this.NewStorageWithPolicy(policy)
storage = this.NewStorageWithPolicy(policy)
if storage == nil {
remotelogs.Error("CACHE", "can not find storage type '"+policy.Type+"'")
continue

View File

@@ -0,0 +1,33 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"io"
"os"
)
type OpenFile struct {
fp *os.File
meta []byte
header []byte
version int64
}
func NewOpenFile(fp *os.File, meta []byte, header []byte) *OpenFile {
return &OpenFile{
fp: fp,
meta: meta,
header: header,
}
}
func (this *OpenFile) SeekStart() error {
_, err := this.fp.Seek(0, io.SeekStart)
return err
}
func (this *OpenFile) Close() error {
this.meta = nil
return this.fp.Close()
}

View File

@@ -0,0 +1,161 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
"github.com/fsnotify/fsnotify"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"path/filepath"
"sync"
"time"
)
type OpenFileCache struct {
poolMap map[string]*OpenFilePool // file path => Pool
poolList *linkedlist.List
watcher *fsnotify.Watcher
locker sync.Mutex
maxSize int
count int
}
func NewOpenFileCache(maxSize int) (*OpenFileCache, error) {
if maxSize <= 0 {
maxSize = 16384
}
var cache = &OpenFileCache{
maxSize: maxSize,
poolMap: map[string]*OpenFilePool{},
poolList: linkedlist.NewList(),
}
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
cache.watcher = watcher
goman.New(func() {
for event := range watcher.Events {
if event.Op&fsnotify.Chmod != fsnotify.Chmod {
cache.Close(event.Name)
}
}
})
return cache, nil
}
func (this *OpenFileCache) Get(filename string) *OpenFile {
this.locker.Lock()
defer this.locker.Unlock()
pool, ok := this.poolMap[filename]
if ok {
file, consumed := pool.Get()
if consumed {
this.count--
}
return file
}
return nil
}
func (this *OpenFileCache) Put(filename string, file *OpenFile) {
this.locker.Lock()
defer this.locker.Unlock()
pool, ok := this.poolMap[filename]
var success bool
if ok {
success = pool.Put(file)
} else {
_ = this.watcher.Add(filename)
pool = NewOpenFilePool(filename)
this.poolMap[filename] = pool
success = pool.Put(file)
}
this.poolList.Push(pool.linkItem)
// 检查长度
if success {
this.count++
// 如果超过当前容量,则关闭最早的
if this.count > this.maxSize {
var delta = this.maxSize / 100 // 清理1%
if delta == 0 {
delta = 1
}
for i := 0; i < delta; i++ {
var head = this.poolList.Head()
if head == nil {
break
}
var headPool = head.Value.(*OpenFilePool)
headFile, consumed := headPool.Get()
if consumed {
this.count--
if headFile != nil {
_ = headFile.Close()
}
}
if headPool.Len() == 0 {
delete(this.poolMap, headPool.filename)
this.poolList.Remove(head)
_ = this.watcher.Remove(headPool.filename)
}
}
}
}
}
func (this *OpenFileCache) Close(filename string) {
this.locker.Lock()
pool, ok := this.poolMap[filename]
if ok {
delete(this.poolMap, filename)
this.poolList.Remove(pool.linkItem)
_ = this.watcher.Remove(filename)
this.count -= pool.Len()
}
this.locker.Unlock()
// 在locker之外提升性能
if ok {
pool.Close()
}
}
func (this *OpenFileCache) CloseAll() {
this.locker.Lock()
for _, pool := range this.poolMap {
pool.Close()
}
this.poolMap = map[string]*OpenFilePool{}
this.poolList.Reset()
_ = this.watcher.Close()
this.locker.Unlock()
}
func (this *OpenFileCache) Debug() {
var ticker = time.NewTicker(5 * time.Second)
goman.New(func() {
for range ticker.C {
logs.Println("==== " + types.String(this.count) + " ====")
this.poolList.Range(func(item *linkedlist.Item) (goNext bool) {
logs.Println(filepath.Base(item.Value.(*OpenFilePool).Filename()), item.Value.(*OpenFilePool).Len())
return true
})
}
})
}

View File

@@ -0,0 +1,76 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
)
type OpenFilePool struct {
c chan *OpenFile
linkItem *linkedlist.Item
filename string
version int64
}
func NewOpenFilePool(filename string) *OpenFilePool {
var pool = &OpenFilePool{
filename: filename,
c: make(chan *OpenFile, 1024),
version: utils.UnixTimeMilli(),
}
pool.linkItem = linkedlist.NewItem(pool)
return pool
}
func (this *OpenFilePool) Filename() string {
return this.filename
}
func (this *OpenFilePool) Get() (*OpenFile, bool) {
select {
case file := <-this.c:
err := file.SeekStart()
if err != nil {
_ = file.Close()
return nil, true
}
file.version = this.version
return file, true
default:
return nil, false
}
}
func (this *OpenFilePool) Put(file *OpenFile) bool {
if file.version > 0 && file.version != this.version {
_ = file.Close()
return false
}
select {
case this.c <- file:
return true
default:
// 多余的直接关闭
_ = file.Close()
return false
}
}
func (this *OpenFilePool) Len() int {
return len(this.c)
}
func (this *OpenFilePool) Close() {
Loop:
for {
select {
case file := <-this.c:
_ = file.Close()
default:
break Loop
}
}
}

View File

@@ -0,0 +1,17 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"testing"
)
func TestOpenFilePool_Get(t *testing.T) {
var pool = caches.NewOpenFilePool("a")
t.Log(pool.Filename())
t.Log(pool.Get())
t.Log(pool.Put(caches.NewOpenFile(nil, nil, []byte{})))
t.Log(pool.Get())
t.Log(pool.Get())
}

View File

@@ -0,0 +1,143 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"encoding/json"
)
// PartialRanges 内容分区范围定义
type PartialRanges struct {
ranges [][2]int64
}
// NewPartialRanges 获取新对象
func NewPartialRanges() *PartialRanges {
return &PartialRanges{ranges: [][2]int64{}}
}
// NewPartialRangesFromJSON 从JSON中解析范围
func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
var rs = [][2]int64{}
err := json.Unmarshal(data, &rs)
if err != nil {
return nil, err
}
var r = NewPartialRanges()
r.ranges = rs
return r, nil
}
// Add 添加新范围
func (this *PartialRanges) Add(begin int64, end int64) {
if begin > end {
begin, end = end, begin
}
var nr = [2]int64{begin, end}
var count = len(this.ranges)
if count == 0 {
this.ranges = [][2]int64{nr}
return
}
// insert
// TODO 将来使用二分法改进
var index = -1
for i, r := range this.ranges {
if r[0] > begin || (r[0] == begin && r[1] >= end) {
index = i
this.ranges = append(this.ranges, [2]int64{})
copy(this.ranges[index+1:], this.ranges[index:])
this.ranges[index] = nr
break
}
}
if index == -1 {
index = count
this.ranges = append(this.ranges, nr)
}
this.merge(index)
}
// Ranges 获取所有范围
func (this *PartialRanges) Ranges() [][2]int64 {
return this.ranges
}
// Contains 检查是否包含某个范围
func (this *PartialRanges) Contains(begin int64, end int64) bool {
if len(this.ranges) == 0 {
return true
}
// TODO 使用二分法查找改进性能
for _, r2 := range this.ranges {
if r2[0] <= begin && r2[1] >= end {
return true
}
}
return false
}
// AsJSON 转换为JSON
func (this *PartialRanges) AsJSON() ([]byte, error) {
return json.Marshal(this.ranges)
}
func (this *PartialRanges) merge(index int) {
// forward
var lastIndex = index
for i := index; i >= 1; i-- {
var curr = this.ranges[i]
var prev = this.ranges[i-1]
var w1 = this.w(curr)
var w2 = this.w(prev)
if w1+w2 >= this.max(curr[1], prev[1])-this.min(curr[0], prev[0])-1 {
prev = [2]int64{this.min(curr[0], prev[0]), this.max(curr[1], prev[1])}
this.ranges[i-1] = prev
this.ranges = append(this.ranges[:i], this.ranges[i+1:]...)
lastIndex = i - 1
} else {
break
}
}
// backward
index = lastIndex
for index < len(this.ranges)-1 {
var curr = this.ranges[index]
var next = this.ranges[index+1]
var w1 = this.w(curr)
var w2 = this.w(next)
if w1+w2 >= this.max(curr[1], next[1])-this.min(curr[0], next[0])-1 {
curr = [2]int64{this.min(curr[0], next[0]), this.max(curr[1], next[1])}
this.ranges = append(this.ranges[:index], this.ranges[index+1:]...)
this.ranges[index] = curr
} else {
break
}
}
}
func (this *PartialRanges) w(r [2]int64) int64 {
return r[1] - r[0]
}
func (this *PartialRanges) min(n1 int64, n2 int64) int64 {
if n1 <= n2 {
return n1
}
return n2
}
func (this *PartialRanges) max(n1 int64, n2 int64) int64 {
if n1 >= n2 {
return n1
}
return n2
}

View File

@@ -0,0 +1,124 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/assert"
"github.com/iwind/TeaGo/logs"
"testing"
)
func TestNewPartialRanges(t *testing.T) {
var r = caches.NewPartialRanges()
r.Add(1, 100)
r.Add(50, 300)
r.Add(30, 80)
r.Add(30, 100)
r.Add(30, 400)
r.Add(1000, 10000)
r.Add(200, 1000)
r.Add(200, 10040)
logs.PrintAsJSON(r.Ranges())
}
func TestNewPartialRanges1(t *testing.T) {
var a = assert.NewAssertion(t)
var r = caches.NewPartialRanges()
r.Add(1, 100)
r.Add(1, 101)
r.Add(1, 102)
r.Add(2, 103)
r.Add(200, 300)
r.Add(1, 1000)
var rs = r.Ranges()
logs.PrintAsJSON(rs, t)
a.IsTrue(len(rs) == 1)
if len(rs) == 1 {
a.IsTrue(rs[0][0] == 1)
a.IsTrue(rs[0][1] == 1000)
}
}
func TestNewPartialRanges2(t *testing.T) {
// low -> high
var r = caches.NewPartialRanges()
r.Add(1, 100)
r.Add(1, 101)
r.Add(1, 102)
r.Add(2, 103)
r.Add(200, 300)
r.Add(301, 302)
r.Add(303, 304)
r.Add(250, 400)
var rs = r.Ranges()
logs.PrintAsJSON(rs, t)
}
func TestNewPartialRanges3(t *testing.T) {
// high -> low
var r = caches.NewPartialRanges()
r.Add(301, 302)
r.Add(303, 304)
r.Add(200, 300)
r.Add(250, 400)
var rs = r.Ranges()
logs.PrintAsJSON(rs, t)
}
func TestNewPartialRanges4(t *testing.T) {
// nearby
var r = caches.NewPartialRanges()
r.Add(301, 302)
r.Add(303, 304)
r.Add(305, 306)
r.Add(417, 417)
r.Add(410, 415)
r.Add(400, 409)
var rs = r.Ranges()
logs.PrintAsJSON(rs, t)
t.Log(r.Contains(400, 416))
}
func TestNewPartialRanges5(t *testing.T) {
var r = caches.NewPartialRanges()
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
}
logs.PrintAsJSON(r.Ranges(), t)
}
func TestNewPartialRanges_AsJSON(t *testing.T) {
var r = caches.NewPartialRanges()
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
}
data, err := r.AsJSON()
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
r2, err := caches.NewPartialRangesFromJSON(data)
if err != nil {
t.Fatal(err)
}
t.Log(r2.Ranges())
}
func BenchmarkNewPartialRanges(b *testing.B) {
for i := 0; i < b.N; i++ {
var r = caches.NewPartialRanges()
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
}
}
}

View File

@@ -24,6 +24,9 @@ type Reader interface {
// ReadBody 读取Body
ReadBody(buf []byte, callback ReaderFunc) error
// Read 实现io.Reader接口
Read(buf []byte) (int, error)
// ReadBodyRange 读取某个范围内的Body
ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error

View File

@@ -11,6 +11,12 @@ import (
type FileReader struct {
fp *os.File
openFile *OpenFile
openFileCache *OpenFileCache
meta []byte
header []byte
expiresAt int64
status int
headerOffset int64
@@ -18,8 +24,7 @@ type FileReader struct {
bodySize int64
bodyOffset int64
bodyBufLen int
bodyBuf []byte
isClosed bool
}
func NewFileReader(fp *os.File) *FileReader {
@@ -27,21 +32,36 @@ func NewFileReader(fp *os.File) *FileReader {
}
func (this *FileReader) Init() error {
return this.InitAutoDiscard(true)
}
func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
if this.openFile != nil {
this.meta = this.openFile.meta
this.header = this.openFile.header
}
isOk := false
defer func() {
if !isOk {
_ = this.discard()
}
}()
var buf = make([]byte, SizeMeta)
ok, err := this.readToBuff(this.fp, buf)
if err != nil {
return err
if autoDiscard {
defer func() {
if !isOk {
_ = this.discard()
}
}()
}
if !ok {
return ErrNotFound
var buf = this.meta
if len(buf) == 0 {
buf = make([]byte, SizeMeta)
ok, err := this.readToBuff(this.fp, buf)
if err != nil {
return err
}
if !ok {
return ErrNotFound
}
this.meta = buf
}
this.expiresAt = int64(binary.BigEndian.Uint32(buf[:SizeExpiresAt]))
@@ -64,13 +84,28 @@ func (this *FileReader) Init() error {
this.headerOffset = int64(SizeMeta) + int64(urlLength)
// body
this.bodyOffset = this.headerOffset + int64(headerSize)
bodySize := int(binary.BigEndian.Uint64(buf[SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength : SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength+SizeBodyLength]))
if bodySize == 0 {
isOk = true
return nil
}
this.bodySize = int64(bodySize)
this.bodyOffset = this.headerOffset + int64(headerSize)
// read header
if this.openFileCache != nil && len(this.header) == 0 {
if headerSize > 0 && headerSize <= 512 {
this.header = make([]byte, headerSize)
_, err := this.fp.Seek(this.headerOffset, io.SeekStart)
if err != nil {
return err
}
_, err = this.readToBuff(this.fp, this.header)
if err != nil {
return err
}
}
}
isOk = true
@@ -106,6 +141,22 @@ func (this *FileReader) BodySize() int64 {
}
func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
// 使用缓存
if len(this.header) > 0 && len(buf) >= len(this.header) {
copy(buf, this.header)
_, err := callback(len(this.header))
if err != nil {
return err
}
// 移动到Body位置
_, err = this.fp.Seek(this.bodyOffset, io.SeekStart)
if err != nil {
return err
}
return nil
}
isOk := false
defer func() {
@@ -135,10 +186,6 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
}
headerSize -= n
} else {
if n > headerSize {
this.bodyBuf = buf[headerSize:]
this.bodyBufLen = n - headerSize
}
_, e := callback(headerSize)
if e != nil {
isOk = true
@@ -157,6 +204,12 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
isOk = true
// 移动到Body位置
_, err = this.fp.Seek(this.bodyOffset, io.SeekStart)
if err != nil {
return err
}
return nil
}
@@ -169,27 +222,7 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
}
}()
offset := this.bodyOffset
// 直接返回从Header中剩余的
if this.bodyBufLen > 0 && len(buf) >= this.bodyBufLen {
offset += int64(this.bodyBufLen)
copy(buf, this.bodyBuf)
isOk = true
goNext, err := callback(this.bodyBufLen)
if err != nil {
return err
}
if !goNext {
return nil
}
if this.bodySize <= int64(this.bodyBufLen) {
return nil
}
}
var offset = this.bodyOffset
// 开始读Body部分
_, err := this.fp.Seek(offset, io.SeekStart)
@@ -222,6 +255,14 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
return nil
}
func (this *FileReader) Read(buf []byte) (n int, err error) {
n, err = this.fp.Read(buf)
if err != nil && err != io.EOF {
_ = this.discard()
}
return
}
func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error {
isOk := false
@@ -292,6 +333,19 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
}
func (this *FileReader) Close() error {
if this.openFileCache != nil {
if this.isClosed {
return nil
}
this.isClosed = true
if this.openFile != nil {
this.openFileCache.Put(this.fp.Name(), this.openFile)
} else {
this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, this.meta, this.header))
}
return nil
}
return this.fp.Close()
}
@@ -306,5 +360,6 @@ func (this *FileReader) readToBuff(fp *os.File, buf []byte) (ok bool, err error)
func (this *FileReader) discard() error {
_ = this.fp.Close()
this.isClosed = true
return os.Remove(this.fp.Name())
}

View File

@@ -54,6 +54,30 @@ func TestFileReader(t *testing.T) {
}
}
func TestFileReader_ReadHeader(t *testing.T) {
var path = "/Users/WorkSpace/EdgeProject/EdgeCache/p43/12/6b/126bbed90fc80f2bdfb19558948b0d49.cache"
fp, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer func() {
_ = fp.Close()
}()
var reader = NewFileReader(fp)
err = reader.Init()
if err != nil {
t.Fatal(err)
}
var buf = make([]byte, 16*1024)
err = reader.ReadHeader(buf, func(n int) (goNext bool, err error) {
t.Log("header:", string(buf[:n]))
return
})
if err != nil {
t.Fatal(err)
}
}
func TestFileReader_Range(t *testing.T) {
storage := NewFileStorage(&serverconfigs.HTTPCachePolicy{
Id: 1,

View File

@@ -2,10 +2,13 @@ package caches
import (
"errors"
"io"
)
type MemoryReader struct {
item *MemoryItem
offset int
}
func NewMemoryReader(item *MemoryItem) *MemoryReader {
@@ -111,6 +114,33 @@ func (this *MemoryReader) ReadBody(buf []byte, callback ReaderFunc) error {
return nil
}
func (this *MemoryReader) Read(buf []byte) (n int, err error) {
bufLen := len(buf)
if bufLen == 0 {
return 0, errors.New("using empty buffer")
}
bodySize := len(this.item.BodyValue)
left := bodySize - this.offset
if bufLen <= left {
copy(buf, this.item.BodyValue[this.offset:this.offset+bufLen])
n = bufLen
this.offset += bufLen
if this.offset >= bodySize {
err = io.EOF
return
}
return
} else {
copy(buf, this.item.BodyValue[this.offset:])
n = left
err = io.EOF
return
}
}
func (this *MemoryReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error {
offset := start
bodySize := int64(len(this.item.BodyValue))

View File

@@ -14,12 +14,12 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"golang.org/x/text/language"
"golang.org/x/text/message"
"io"
"math"
"os"
"path/filepath"
@@ -47,6 +47,9 @@ const (
HotItemSize = 1024
)
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
var sharedWritingFileKeyLocker = sync.Mutex{}
// FileStorage 文件缓存
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
@@ -56,23 +59,23 @@ type FileStorage struct {
memoryStorage *MemoryStorage // 一级缓存
totalSize int64
list ListInterface
writingKeyMap map[string]zero.Zero // key => bool
locker sync.RWMutex
purgeTicker *utils.Ticker
list ListInterface
locker sync.RWMutex
purgeTicker *utils.Ticker
hotMap map[string]*HotItem // key => count
hotMapLocker sync.Mutex
lastHotSize int
hotTicker *utils.Ticker
openFileCache *OpenFileCache
}
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
return &FileStorage{
policy: policy,
writingKeyMap: map[string]zero.Zero{},
hotMap: map[string]*HotItem{},
lastHotSize: -1,
policy: policy,
hotMap: map[string]*HotItem{},
lastHotSize: -1,
}
}
@@ -99,13 +102,13 @@ func (this *FileStorage) Init() error {
return err
}
this.cacheConfig = cacheConfig
cacheDir := cacheConfig.Dir
if !filepath.IsAbs(this.cacheConfig.Dir) {
this.cacheConfig.Dir = Tea.Root + Tea.DS + this.cacheConfig.Dir
}
dir := this.cacheConfig.Dir
this.cacheConfig.Dir = filepath.Clean(this.cacheConfig.Dir)
var dir = this.cacheConfig.Dir
if len(dir) == 0 {
return errors.New("[CACHE]cache storage dir can not be empty")
@@ -158,7 +161,7 @@ func (this *FileStorage) Init() error {
} else if size > 1024 {
sizeMB = fmt.Sprintf("%.3f K", float64(size)/1024)
}
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+cacheDir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+this.cacheConfig.Dir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
}()
// 初始化list
@@ -201,6 +204,15 @@ func (this *FileStorage) Init() error {
}
}
// open file cache
if this.cacheConfig.OpenFileCache != nil && this.cacheConfig.OpenFileCache.IsOn && this.cacheConfig.OpenFileCache.Max > 0 {
this.openFileCache, err = NewOpenFileCache(this.cacheConfig.OpenFileCache.Max)
logs.Println("start open file cache")
if err != nil {
remotelogs.Error("CACHE", "open file cache failed: "+err.Error())
}
}
return nil
}
@@ -237,12 +249,22 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
// TODO 尝试使用mmap加快读取速度
var isOk = false
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
var openFile *OpenFile
if this.openFileCache != nil {
openFile = this.openFileCache.Get(path)
}
var fp *os.File
var err error
if openFile == nil {
fp, err = os.OpenFile(path, os.O_RDONLY, 0444)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
return nil, ErrNotFound
}
return nil, ErrNotFound
} else {
fp = openFile.fp
}
defer func() {
if !isOk {
@@ -251,10 +273,9 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
}
}()
reader := NewFileReader(fp)
if err != nil {
return nil, err
}
var reader = NewFileReader(fp)
reader.openFile = openFile
reader.openFileCache = this.openFileCache
err = reader.Init()
if err != nil {
return nil, err
@@ -304,31 +325,31 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
}
// OpenWriter 打开缓存文件等待写入
func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Writer, error) {
func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, size int64, isPartial bool) (Writer, error) {
// 先尝试内存缓存
if this.memoryStorage != nil {
writer, err := this.memoryStorage.OpenWriter(key, expiredAt, status)
// 我们限定仅小文件优先存在内存中
if !isPartial && this.memoryStorage != nil && size > 0 && size < 32*1024*1024 {
writer, err := this.memoryStorage.OpenWriter(key, expiredAt, status, size, false)
if err == nil {
return writer, nil
}
}
// 是否正在写入
var isWriting = false
this.locker.Lock()
_, ok := this.writingKeyMap[key]
this.locker.Unlock()
var isOk = false
sharedWritingFileKeyLocker.Lock()
_, ok := sharedWritingFileKeyMap[key]
if ok {
sharedWritingFileKeyLocker.Unlock()
return nil, ErrFileIsWriting
}
this.locker.Lock()
this.writingKeyMap[key] = zero.New()
this.locker.Unlock()
sharedWritingFileKeyMap[key] = zero.New()
sharedWritingFileKeyLocker.Unlock()
defer func() {
if !isWriting {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
if !isOk {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
sharedWritingFileKeyLocker.Unlock()
}
}()
@@ -340,13 +361,13 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
if this.policy.MaxKeys > 0 && count > this.policy.MaxKeys {
return nil, NewCapacityError("write file cache failed: too many keys in cache storage")
}
capacityBytes := this.diskCapacityBytes()
var capacityBytes = this.diskCapacityBytes()
if capacityBytes > 0 && capacityBytes <= this.totalSize {
return nil, NewCapacityError("write file cache failed: over disk size, current total size: " + strconv.FormatInt(this.totalSize, 10) + " bytes, capacity: " + strconv.FormatInt(capacityBytes, 10))
}
hash := stringutil.Md5(key)
dir := this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
var hash = stringutil.Md5(key)
var dir = this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
_, err = os.Stat(dir)
if err != nil {
if !os.IsNotExist(err) {
@@ -358,21 +379,30 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
}
}
// 检查缓存是否已经生成
var cachePath = dir + "/" + hash + ".cache"
stat, err := os.Stat(cachePath)
if err == nil && time.Now().Sub(stat.ModTime()) <= 1*time.Second {
// 防止并发连续写入
return nil, ErrFileIsWriting
}
var tmpPath = cachePath + ".tmp"
if isPartial {
tmpPath = cachePath
}
// 先删除
err = this.list.Remove(hash)
if err != nil {
return nil, err
}
path := dir + "/" + hash + ".cache.tmp"
writer, err := os.OpenFile(path, os.O_CREATE|os.O_SYNC|os.O_WRONLY, 0666)
writer, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_SYNC|os.O_WRONLY, 0666)
if err != nil {
return nil, err
}
isWriting = true
isOk := false
removeOnFailure := true
var removeOnFailure = true
defer func() {
if err != nil {
isOk = false
@@ -382,7 +412,7 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
if !isOk {
_ = writer.Close()
if removeOnFailure {
_ = os.Remove(path)
_ = os.Remove(tmpPath)
}
}
}()
@@ -394,71 +424,96 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
return nil, ErrFileIsWriting
}
err = writer.Truncate(0)
if err != nil {
return nil, err
}
// 写入过期时间
bytes4 := make([]byte, 4)
{
binary.BigEndian.PutUint32(bytes4, uint32(expiredAt))
_, err = writer.Write(bytes4)
if err != nil {
return nil, err
// 是否已经有内容
var isNewCreated = true
var partialBodyOffset int64
if isPartial {
partialFP, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
if err == nil {
var partialReader = NewFileReader(partialFP)
err = partialReader.InitAutoDiscard(false)
if err == nil && partialReader.bodyOffset > 0 {
isNewCreated = false
partialBodyOffset = partialReader.bodyOffset
}
_ = partialReader.Close()
}
}
// 写入状态码
if status > 999 || status < 100 {
status = 200
}
_, err = writer.WriteString(strconv.Itoa(status))
if err != nil {
return nil, err
}
// 写入URL长度
{
binary.BigEndian.PutUint32(bytes4, uint32(len(key)))
_, err = writer.Write(bytes4)
if isNewCreated {
err = writer.Truncate(0)
if err != nil {
return nil, err
}
}
// 写入Header Length
{
binary.BigEndian.PutUint32(bytes4, uint32(0))
_, err = writer.Write(bytes4)
// 写入过期时间
bytes4 := make([]byte, 4)
{
binary.BigEndian.PutUint32(bytes4, uint32(expiredAt))
_, err = writer.Write(bytes4)
if err != nil {
return nil, err
}
}
// 写入状态码
if status > 999 || status < 100 {
status = 200
}
_, err = writer.WriteString(strconv.Itoa(status))
if err != nil {
return nil, err
}
}
// 写入Body Length
{
b := make([]byte, SizeBodyLength)
binary.BigEndian.PutUint64(b, uint64(0))
_, err = writer.Write(b)
// 写入URL长度
{
binary.BigEndian.PutUint32(bytes4, uint32(len(key)))
_, err = writer.Write(bytes4)
if err != nil {
return nil, err
}
}
// 写入Header Length
{
binary.BigEndian.PutUint32(bytes4, uint32(0))
_, err = writer.Write(bytes4)
if err != nil {
return nil, err
}
}
// 写入Body Length
{
b := make([]byte, SizeBodyLength)
binary.BigEndian.PutUint64(b, uint64(0))
_, err = writer.Write(b)
if err != nil {
return nil, err
}
}
// 写入URL
_, err = writer.WriteString(key)
if err != nil {
return nil, err
}
}
// 写入URL
_, err = writer.WriteString(key)
if err != nil {
return nil, err
}
isOk = true
return NewFileWriter(writer, key, expiredAt, func() {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
}), nil
if isPartial {
return NewPartialFileWriter(writer, key, expiredAt, isNewCreated, isPartial, partialBodyOffset, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
sharedWritingFileKeyLocker.Unlock()
}), nil
} else {
return NewFileWriter(writer, key, expiredAt, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
sharedWritingFileKeyLocker.Unlock()
}), nil
}
}
// AddToList 添加到List
@@ -618,6 +673,8 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
// Stop 停止
func (this *FileStorage) Stop() {
events.Remove(this)
this.locker.Lock()
defer this.locker.Unlock()
@@ -635,6 +692,10 @@ func (this *FileStorage) Stop() {
}
_ = this.list.Close()
if this.openFileCache != nil {
this.openFileCache.CloseAll()
}
}
// TotalDiskSize 消耗的磁盘尺寸
@@ -697,11 +758,20 @@ func (this *FileStorage) initList() error {
}
}
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
events.On(events.EventQuit, func() {
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("CACHE", "quit clean timer")
var ticker = this.purgeTicker
if ticker != nil {
ticker.Stop()
{
var ticker = this.purgeTicker
if ticker != nil {
ticker.Stop()
}
}
{
var ticker = this.hotTicker
if ticker != nil {
ticker.Stop()
}
}
})
goman.New(func() {
@@ -728,98 +798,6 @@ func (this *FileStorage) initList() error {
return nil
}
// 解析文件信息
func (this *FileStorage) decodeFile(path string) (*Item, error) {
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
if err != nil {
return nil, err
}
isAllOk := false
defer func() {
_ = fp.Close()
if !isAllOk {
_ = os.Remove(path)
}
}()
item := &Item{
Type: ItemTypeFile,
MetaSize: SizeMeta,
}
bytes4 := make([]byte, 4)
// 过期时间
ok, err := this.readToBuff(fp, bytes4)
if err != nil {
return nil, err
}
if !ok {
return nil, ErrNotFound
}
item.ExpiredAt = int64(binary.BigEndian.Uint32(bytes4))
// 是否已过期
if item.ExpiredAt < time.Now().Unix() {
return nil, ErrNotFound
}
// URL Size
_, err = fp.Seek(int64(SizeExpiresAt+SizeStatus), io.SeekStart)
if err != nil {
return nil, err
}
ok, err = this.readToBuff(fp, bytes4)
if err != nil {
return nil, err
}
if !ok {
return nil, ErrNotFound
}
urlSize := binary.BigEndian.Uint32(bytes4)
// Header Size
ok, err = this.readToBuff(fp, bytes4)
if err != nil {
return nil, err
}
if !ok {
return nil, ErrNotFound
}
item.HeaderSize = int64(binary.BigEndian.Uint32(bytes4))
// Body Size
bytes8 := make([]byte, 8)
ok, err = this.readToBuff(fp, bytes8)
if err != nil {
return nil, err
}
if !ok {
return nil, ErrNotFound
}
item.BodySize = int64(binary.BigEndian.Uint64(bytes8))
// URL
if urlSize > 0 {
data := utils.BytePool1k.Get()
result, ok, err := this.readN(fp, data, int(urlSize))
utils.BytePool1k.Put(data)
if err != nil {
return nil, err
}
if !ok {
return nil, ErrNotFound
}
item.Key = string(result)
}
isAllOk = true
return item, nil
}
// 清理任务
func (this *FileStorage) purgeLoop() {
// 计算是否应该开启LFU清理
@@ -956,7 +934,7 @@ func (this *FileStorage) hotLoop() {
continue
}
writer, err := this.memoryStorage.openWriter(item.Key, item.ExpiresAt, item.Status, false)
writer, err := this.memoryStorage.openWriter(item.Key, item.ExpiresAt, item.Status, reader.BodySize(), false)
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "transfer hot item failed: "+err.Error())
@@ -981,6 +959,9 @@ func (this *FileStorage) hotLoop() {
err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
_, err = writer.Write(buf[:n])
if err == nil {
goNext = true
}
return
})
if err != nil {
@@ -1003,34 +984,6 @@ func (this *FileStorage) hotLoop() {
}
}
func (this *FileStorage) readToBuff(fp *os.File, buf []byte) (ok bool, err error) {
n, err := fp.Read(buf)
if err != nil {
return false, err
}
ok = n == len(buf)
return
}
func (this *FileStorage) readN(fp *os.File, buf []byte, total int) (result []byte, ok bool, err error) {
for {
n, err := fp.Read(buf)
if err != nil {
return nil, false, err
}
if n > 0 {
if n >= total {
result = append(result, buf[:total]...)
ok = true
return result, ok, nil
} else {
total -= n
result = append(result, buf[:n]...)
}
}
}
}
func (this *FileStorage) diskCapacityBytes() int64 {
c1 := this.policy.CapacityBytes()
if SharedManager.MaxDiskCapacity != nil {

View File

@@ -62,7 +62,7 @@ func TestFileStorage_OpenWriter(t *testing.T) {
header := []byte("Header")
body := []byte("This is Body")
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200)
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -87,6 +87,41 @@ func TestFileStorage_OpenWriter(t *testing.T) {
t.Log("ok")
}
func TestFileStorage_OpenWriter_Partial(t *testing.T) {
var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
Id: 2,
IsOn: true,
Options: map[string]interface{}{
"dir": Tea.Root + "/caches",
},
})
err := storage.Init()
if err != nil {
t.Fatal(err)
}
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, true)
if err != nil {
t.Fatal(err)
}
_, err = writer.WriteHeader([]byte("Content-Type:text/html; charset=utf-8"))
if err != nil {
t.Fatal(err)
}
err = writer.WriteAt([]byte("Hello, World"), 0)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log(writer)
}
func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
storage := NewFileStorage(&serverconfigs.HTTPCachePolicy{
Id: 1,
@@ -104,7 +139,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
t.Log(time.Since(now).Seconds()*1000, "ms")
}()
writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200)
writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -177,10 +212,11 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
go func(i int) {
defer wg.Done()
writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200)
writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, false)
if err != nil {
if err != ErrFileIsWriting {
t.Fatal(err)
t.Error(err)
return
}
return
}
@@ -188,7 +224,8 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
_, err = writer.Write([]byte("Hello,World"))
if err != nil {
t.Fatal(err)
t.Error(err)
return
}
// 故意造成慢速写入
@@ -196,7 +233,8 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
err = writer.Close()
if err != nil {
t.Fatal(err)
t.Error(err)
return
}
}(i)
}
@@ -229,10 +267,11 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
go func(i int) {
defer wg.Done()
writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200)
writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, false)
if err != nil {
if err != ErrFileIsWriting {
t.Fatal(err)
t.Error(err)
return
}
return
}
@@ -241,7 +280,8 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
t.Log("writing")
_, err = writer.Write([]byte("Hello,World"))
if err != nil {
t.Fatal(err)
t.Error(err)
return
}
// 故意造成慢速写入
@@ -249,7 +289,8 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
err = writer.Close()
if err != nil {
t.Fatal(err)
t.Error(err)
return
}
}(i)
}
@@ -270,7 +311,7 @@ func TestFileStorage_Read(t *testing.T) {
t.Fatal(err)
}
now := time.Now()
reader, err := storage.OpenReader("my-key")
reader, err := storage.OpenReader("my-key", false)
if err != nil {
t.Fatal(err)
}
@@ -306,7 +347,7 @@ func TestFileStorage_Read_HTTP_Response(t *testing.T) {
t.Fatal(err)
}
now := time.Now()
reader, err := storage.OpenReader("my-http-response")
reader, err := storage.OpenReader("my-http-response", false)
if err != nil {
t.Fatal(err)
}
@@ -360,7 +401,7 @@ func TestFileStorage_Read_NotFound(t *testing.T) {
}
now := time.Now()
buf := make([]byte, 6)
reader, err := storage.OpenReader("my-key-10000")
reader, err := storage.OpenReader("my-key-10000", false)
if err != nil {
if err == ErrNotFound {
t.Log("cache not fund")
@@ -482,11 +523,7 @@ func TestFileStorage_DecodeFile(t *testing.T) {
t.Fatal(err)
}
_, path := storage.keyPath("my-key")
item, err := storage.decodeFile(path)
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(item, t)
t.Log(path)
}
func BenchmarkFileStorage_Read(b *testing.B) {
@@ -506,7 +543,7 @@ func BenchmarkFileStorage_Read(b *testing.B) {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
reader, err := storage.OpenReader("my-key")
reader, err := storage.OpenReader("my-key", false)
if err != nil {
b.Fatal(err)
}

View File

@@ -13,7 +13,7 @@ type StorageInterface interface {
OpenReader(key string, useStale bool) (reader Reader, err error)
// OpenWriter 打开缓存写入器等待写入
OpenWriter(key string, expiredAt int64, status int) (Writer, error)
OpenWriter(key string, expiredAt int64, status int, size int64, isPartial bool) (Writer, error)
// Delete 删除某个键值对应的缓存
Delete(key string) error

View File

@@ -145,11 +145,15 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool) (Reader, error)
}
// OpenWriter 打开缓存写入器等待写入
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int) (Writer, error) {
return this.openWriter(key, expiredAt, status, true)
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, size int64, isPartial bool) (Writer, error) {
// TODO 内存缓存暂时不支持分块内容存储
if isPartial {
return nil, ErrFileIsWriting
}
return this.openWriter(key, expiredAt, status, size, true)
}
func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, isDirty bool) (Writer, error) {
func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, size int64, isDirty bool) (Writer, error) {
this.locker.Lock()
defer this.locker.Unlock()
@@ -182,7 +186,10 @@ func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, i
return nil, NewCapacityError("write memory cache failed: too many keys in cache storage")
}
capacityBytes := this.memoryCapacityBytes()
if capacityBytes > 0 && capacityBytes <= this.totalSize {
if size < 0 {
size = 0
}
if capacityBytes > 0 && capacityBytes <= this.totalSize+size {
return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
}
@@ -384,7 +391,7 @@ func (this *MemoryStorage) flushItem(key string) {
return
}
writer, err := this.parentStorage.OpenWriter(key, item.ExpiredAt, item.Status)
writer, err := this.parentStorage.OpenWriter(key, item.ExpiredAt, item.Status, -1, false)
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "flush items failed: open writer failed: "+err.Error())

View File

@@ -15,7 +15,7 @@ import (
func TestMemoryStorage_OpenWriter(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -25,7 +25,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
t.Log(storage.valuesMap)
{
reader, err := storage.OpenReader("abc")
reader, err := storage.OpenReader("abc", false)
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc")
@@ -52,7 +52,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
}
{
_, err := storage.OpenReader("abc 2")
_, err := storage.OpenReader("abc 2", false)
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc2")
@@ -62,13 +62,13 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
}
}
writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200)
writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello123"))
{
reader, err := storage.OpenReader("abc")
reader, err := storage.OpenReader("abc", false)
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc")
@@ -97,13 +97,13 @@ func TestMemoryStorage_OpenReaderLock(t *testing.T) {
IsDone: true,
},
}
_, _ = storage.OpenReader("test")
_, _ = storage.OpenReader("test", false)
}
func TestMemoryStorage_Delete(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
{
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -111,7 +111,7 @@ func TestMemoryStorage_Delete(t *testing.T) {
t.Log(len(storage.valuesMap))
}
{
writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200)
writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -126,7 +126,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -139,7 +139,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -163,7 +163,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -175,7 +175,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -198,7 +198,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -210,7 +210,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -241,7 +241,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
for i := 0; i < 1000; i++ {
expiredAt := time.Now().Unix() + int64(rands.Int(0, 60))
key := "abc" + strconv.Itoa(i)
writer, err := storage.OpenWriter(key, expiredAt, 200)
writer, err := storage.OpenWriter(key, expiredAt, 200, -1, false)
if err != nil {
t.Fatal(err)
}

View File

@@ -8,6 +8,9 @@ type Writer interface {
// Write 写入Body数据
Write(data []byte) (n int, err error)
// WriteAt 在指定位置写入数据
WriteAt(data []byte, offset int64) error
// HeaderSize 写入的Header数据大小
HeaderSize() int64

View File

@@ -1,76 +0,0 @@
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/compressions"
)
type compressionWriter struct {
rawWriter Writer
writer compressions.Writer
key string
expiredAt int64
}
func NewCompressionWriter(gw Writer, cpWriter compressions.Writer, key string, expiredAt int64) Writer {
return &compressionWriter{
rawWriter: gw,
writer: cpWriter,
key: key,
expiredAt: expiredAt,
}
}
func (this *compressionWriter) WriteHeader(data []byte) (n int, err error) {
return this.writer.Write(data)
}
// WriteHeaderLength 写入Header长度数据
func (this *compressionWriter) WriteHeaderLength(headerLength int) error {
return nil
}
// WriteBodyLength 写入Body长度数据
func (this *compressionWriter) WriteBodyLength(bodyLength int64) error {
return nil
}
func (this *compressionWriter) Write(data []byte) (n int, err error) {
return this.writer.Write(data)
}
func (this *compressionWriter) Close() error {
err := this.writer.Close()
if err != nil {
return err
}
return this.rawWriter.Close()
}
func (this *compressionWriter) Discard() error {
err := this.writer.Close()
if err != nil {
return err
}
return this.rawWriter.Discard()
}
func (this *compressionWriter) Key() string {
return this.key
}
func (this *compressionWriter) ExpiredAt() int64 {
return this.expiredAt
}
func (this *compressionWriter) HeaderSize() int64 {
return this.rawWriter.HeaderSize()
}
func (this *compressionWriter) BodySize() int64 {
return this.rawWriter.BodySize()
}
// ItemType 内容类型
func (this *compressionWriter) ItemType() ItemType {
return this.rawWriter.ItemType()
}

View File

@@ -2,10 +2,12 @@ package caches
import (
"encoding/binary"
"errors"
"github.com/iwind/TeaGo/types"
"io"
"os"
"strings"
"sync"
)
type FileWriter struct {
@@ -15,6 +17,7 @@ type FileWriter struct {
bodySize int64
expiredAt int64
endFunc func()
once sync.Once
}
func NewFileWriter(rawWriter *os.File, key string, expiredAt int64, endFunc func()) *FileWriter {
@@ -63,6 +66,13 @@ func (this *FileWriter) Write(data []byte) (n int, err error) {
return
}
// WriteAt 在指定位置写入数据
func (this *FileWriter) WriteAt(data []byte, offset int64) error {
_ = data
_ = offset
return errors.New("not supported")
}
// WriteBodyLength 写入Body长度数据
func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
bytes8 := make([]byte, 8)
@@ -82,18 +92,25 @@ func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
// Close 关闭
func (this *FileWriter) Close() error {
defer this.endFunc()
defer this.once.Do(func() {
this.endFunc()
})
path := this.rawWriter.Name()
err := this.WriteHeaderLength(types.Int(this.headerSize))
if err != nil {
_ = this.rawWriter.Close()
_ = os.Remove(path)
return err
}
err = this.WriteBodyLength(this.bodySize)
if err != nil {
_ = this.rawWriter.Close()
_ = os.Remove(path)
return err
}
path := this.rawWriter.Name()
err = this.rawWriter.Close()
if err != nil {
_ = os.Remove(path)
@@ -109,7 +126,9 @@ func (this *FileWriter) Close() error {
// Discard 丢弃
func (this *FileWriter) Discard() error {
defer this.endFunc()
defer this.once.Do(func() {
this.endFunc()
})
_ = this.rawWriter.Close()

View File

@@ -1,7 +1,9 @@
package caches
import (
"errors"
"github.com/cespare/xxhash"
"sync"
"time"
)
@@ -18,6 +20,7 @@ type MemoryWriter struct {
hash uint64
item *MemoryItem
endFunc func()
once sync.Once
}
func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, endFunc func()) *MemoryWriter {
@@ -53,6 +56,13 @@ func (this *MemoryWriter) Write(data []byte) (n int, err error) {
return len(data), nil
}
// WriteAt 在指定位置写入数据
func (this *MemoryWriter) WriteAt(b []byte, offset int64) error {
_ = b
_ = offset
return errors.New("not supported")
}
// HeaderSize 数据尺寸
func (this *MemoryWriter) HeaderSize() int64 {
return this.headerSize
@@ -66,7 +76,9 @@ func (this *MemoryWriter) BodySize() int64 {
// Close 关闭
func (this *MemoryWriter) Close() error {
// 需要在Locker之外
defer this.endFunc()
defer this.once.Do(func() {
this.endFunc()
})
if this.item == nil {
return nil
@@ -92,7 +104,9 @@ func (this *MemoryWriter) Close() error {
// Discard 丢弃
func (this *MemoryWriter) Discard() error {
// 需要在Locker之外
defer this.endFunc()
defer this.once.Do(func() {
this.endFunc()
})
this.storage.locker.Lock()
delete(this.storage.valuesMap, this.hash)

View File

@@ -0,0 +1,173 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"encoding/binary"
"github.com/iwind/TeaGo/types"
"io"
"os"
"strings"
"sync"
)
type PartialFileWriter struct {
rawWriter *os.File
key string
headerSize int64
bodySize int64
expiredAt int64
endFunc func()
once sync.Once
isNew bool
isPartial bool
bodyOffset int64
}
func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, isNew bool, isPartial bool, bodyOffset int64, endFunc func()) *PartialFileWriter {
return &PartialFileWriter{
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
endFunc: endFunc,
isNew: isNew,
isPartial: isPartial,
bodyOffset: bodyOffset,
}
}
// WriteHeader 写入数据
func (this *PartialFileWriter) WriteHeader(data []byte) (n int, err error) {
if !this.isNew {
return
}
n, err = this.rawWriter.Write(data)
this.headerSize += int64(n)
if err != nil {
_ = this.Discard()
}
return
}
// WriteHeaderLength 写入Header长度数据
func (this *PartialFileWriter) WriteHeaderLength(headerLength int) error {
bytes4 := make([]byte, 4)
binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength, io.SeekStart)
if err != nil {
_ = this.Discard()
return err
}
_, err = this.rawWriter.Write(bytes4)
if err != nil {
_ = this.Discard()
return err
}
return nil
}
// Write 写入数据
func (this *PartialFileWriter) Write(data []byte) (n int, err error) {
n, err = this.rawWriter.Write(data)
this.bodySize += int64(n)
if err != nil {
_ = this.Discard()
}
return
}
// WriteAt 在指定位置写入数据
func (this *PartialFileWriter) WriteAt(data []byte, offset int64) error {
if this.bodyOffset == 0 {
this.bodyOffset = SizeMeta + int64(len(this.key)) + this.headerSize
}
_, err := this.rawWriter.WriteAt(data, this.bodyOffset+offset)
return err
}
// WriteBodyLength 写入Body长度数据
func (this *PartialFileWriter) WriteBodyLength(bodyLength int64) error {
bytes8 := make([]byte, 8)
binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength, io.SeekStart)
if err != nil {
_ = this.Discard()
return err
}
_, err = this.rawWriter.Write(bytes8)
if err != nil {
_ = this.Discard()
return err
}
return nil
}
// Close 关闭
func (this *PartialFileWriter) Close() error {
defer this.once.Do(func() {
this.endFunc()
})
var path = this.rawWriter.Name()
if this.isNew {
err := this.WriteHeaderLength(types.Int(this.headerSize))
if err != nil {
_ = this.rawWriter.Close()
_ = os.Remove(path)
return err
}
err = this.WriteBodyLength(this.bodySize)
if err != nil {
_ = this.rawWriter.Close()
_ = os.Remove(path)
return err
}
}
err := this.rawWriter.Close()
if err != nil {
_ = os.Remove(path)
} else if !this.isPartial {
err = os.Rename(path, strings.Replace(path, ".tmp", "", 1))
if err != nil {
_ = os.Remove(path)
}
}
return err
}
// Discard 丢弃
func (this *PartialFileWriter) Discard() error {
defer this.once.Do(func() {
this.endFunc()
})
_ = this.rawWriter.Close()
err := os.Remove(this.rawWriter.Name())
return err
}
func (this *PartialFileWriter) HeaderSize() int64 {
return this.headerSize
}
func (this *PartialFileWriter) BodySize() int64 {
return this.bodySize
}
func (this *PartialFileWriter) ExpiredAt() int64 {
return this.expiredAt
}
func (this *PartialFileWriter) Key() string {
return this.key
}
// ItemType 获取内容类型
func (this *PartialFileWriter) ItemType() ItemType {
return ItemTypeFile
}

View File

@@ -0,0 +1,50 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"testing"
"time"
)
func TestPartialFileWriter_SeekOffset(t *testing.T) {
var path = "/tmp/test@partial.cache"
_ = os.Remove(path)
var reader = func() {
data, err := ioutil.ReadFile(path)
if err != nil {
t.Fatal(err)
}
t.Log("["+types.String(len(data))+"]", string(data))
}
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
t.Fatal(err)
}
var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, true, true, 0, func() {
t.Log("end")
})
_, err = writer.WriteHeader([]byte("header"))
if err != nil {
t.Fatal(err)
}
// 移动位置
err = writer.WriteAt([]byte("HELLO"), 100)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader()
}

View File

@@ -5,6 +5,7 @@ package compressions
import (
"github.com/andybalholm/brotli"
"io"
"strings"
)
type BrotliReader struct {
@@ -16,7 +17,11 @@ func NewBrotliReader(reader io.Reader) (Reader, error) {
}
func (this *BrotliReader) Read(p []byte) (n int, err error) {
return this.reader.Read(p)
n, err = this.reader.Read(p)
if err != nil && strings.Contains(err.Error(), "excessive") {
err = io.EOF
}
return
}
func (this *BrotliReader) Close() error {

View File

@@ -0,0 +1,68 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"io"
"os"
"testing"
)
func TestGzipReader(t *testing.T) {
fp, err := os.Open("/Users/WorkSpace/EdgeProject/EdgeCache/p43/36/7e/367e02720713fe05b66573a1d69b4f0a.cache")
if err != nil {
// not fatal
t.Log(err)
return
}
defer func() {
_ = fp.Close()
}()
var buf = make([]byte, 32*1024)
cacheReader := caches.NewFileReader(fp)
err = cacheReader.Init()
if err != nil {
t.Fatal(err)
}
var headerBuf = []byte{}
err = cacheReader.ReadHeader(buf, func(n int) (goNext bool, err error) {
headerBuf = append(headerBuf, buf[:n]...)
for {
nIndex := bytes.Index(headerBuf, []byte{'\n'})
if nIndex >= 0 {
row := headerBuf[:nIndex]
spaceIndex := bytes.Index(row, []byte{':'})
if spaceIndex <= 0 {
return false, errors.New("invalid header '" + string(row) + "'")
}
headerBuf = headerBuf[nIndex+1:]
} else {
break
}
}
return true, nil
})
reader, err := NewGzipReader(cacheReader)
if err != nil {
t.Fatal(err)
}
for {
n, err := reader.Read(buf)
if err != nil {
if err != io.EOF {
t.Fatal(err)
} else {
break
}
}
t.Log(string(buf[:n]))
_ = n
}
}

View File

@@ -18,6 +18,11 @@ const (
var ErrNotSupportedContentEncoding = errors.New("not supported content encoding")
// AllEncodings 当前支持的所有编码
func AllEncodings() []ContentEncoding {
return []ContentEncoding{ContentEncodingBr, ContentEncodingGzip, ContentEncodingDeflate}
}
// NewReader 获取Reader
func NewReader(reader io.Reader, contentEncoding ContentEncoding) (Reader, error) {
switch contentEncoding {

View File

@@ -1,4 +1,5 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build community
// +build community
package teaconst

View File

@@ -1,4 +1,5 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build plus
// +build plus
package teaconst

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.3.7"
Version = "0.4.3"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -2,6 +2,8 @@
package teaconst
import "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
var (
// 流量统计
@@ -10,4 +12,6 @@ var (
NodeId int64 = 0
NodeIdString = ""
GlobalProductName = nodeconfigs.DefaultProductName
)

View File

@@ -1,27 +1,71 @@
package events
import "sync"
import (
"sync"
)
var eventsMap = map[string][]func(){} // event => []callbacks
type Callbacks = []func()
var eventsMap = map[Event]map[interface{}]Callbacks{} // event => map[event key][]callback
var locker = sync.Mutex{}
// 增加事件回调
func On(event string, callback func()) {
var eventKeyId = 0
func NewKey() interface{} {
locker.Lock()
defer locker.Unlock()
eventKeyId++
return eventKeyId
}
// On 增加事件回调
func On(event Event, callback func()) {
OnKey(event, nil, callback)
}
// OnKey 使用Key增加事件回调
func OnKey(event Event, key interface{}, callback func()) {
if key == nil {
key = NewKey()
}
locker.Lock()
defer locker.Unlock()
callbacks, _ := eventsMap[event]
callbacks = append(callbacks, callback)
eventsMap[event] = callbacks
m, ok := eventsMap[event]
if !ok {
m = map[interface{}]Callbacks{}
eventsMap[event] = m
}
m[key] = append(m[key], callback)
}
// 通知事件
func Notify(event string) {
// Remove 删除事件回调
func Remove(key interface{}) {
if key == nil {
return
}
locker.Lock()
callbacks, _ := eventsMap[event]
for k, m := range eventsMap {
_, ok := m[key]
if ok {
delete(m, key)
eventsMap[k] = m
}
}
locker.Unlock()
}
// Notify 通知事件
func Notify(event Event) {
locker.Lock()
m := eventsMap[event]
locker.Unlock()
for _, callback := range callbacks {
callback()
for _, callbacks := range m {
for _, callback := range callbacks {
callback()
}
}
}

View File

@@ -1,16 +1,33 @@
package events
package events_test
import "testing"
import (
"github.com/TeaOSLab/EdgeNode/internal/events"
"testing"
)
func TestOn(t *testing.T) {
On("hello", func() {
type User struct {
name string
}
var u = &User{}
var u2 = &User{}
events.On("hello", func() {
t.Log("world")
})
On("hello", func() {
events.On("hello", func() {
t.Log("world2")
})
On("hello2", func() {
events.OnKey("hello", u, func() {
t.Log("world3")
})
events.OnKey("hello", u, func() {
t.Log("world4")
})
events.Remove(u)
events.Remove(u2)
events.OnKey("hello2", nil, func() {
t.Log("world2")
})
Notify("hello")
events.Notify("hello")
}

View File

@@ -0,0 +1,42 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package firewalls
import (
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
)
var currentFirewall FirewallInterface
// 初始化
func init() {
events.On(events.EventLoaded, func() {
var firewall = Firewall()
if firewall.Name() == "mock" {
remotelogs.Warn("FIREWALL", "'firewalld' on this system should be enabled to block attackers more effectively")
} else {
remotelogs.Println("FIREWALL", "found local firewall '"+firewall.Name()+"'")
}
})
}
// Firewall 查找当前系统中最适合的防火墙
func Firewall() FirewallInterface {
if currentFirewall != nil {
return currentFirewall
}
// firewalld
{
var firewalld = NewFirewalld()
if firewalld.IsReady() {
currentFirewall = firewalld
return currentFirewall
}
}
// 至少返回一个
currentFirewall = NewMockFirewall()
return currentFirewall
}

View File

@@ -0,0 +1,135 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package firewalls
import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
"os/exec"
"strings"
)
type Firewalld struct {
isReady bool
exe string
cmdQueue chan *exec.Cmd
}
func NewFirewalld() *Firewalld {
var firewalld = &Firewalld{
cmdQueue: make(chan *exec.Cmd, 4096),
}
path, err := exec.LookPath("firewall-cmd")
if err == nil && len(path) > 0 {
var cmd = exec.Command(path, "-V")
err := cmd.Run()
if err == nil {
firewalld.exe = path
firewalld.isReady = true
firewalld.init()
}
}
return firewalld
}
func (this *Firewalld) init() {
goman.New(func() {
for cmd := range this.cmdQueue {
err := cmd.Run()
if err != nil {
if strings.HasPrefix(err.Error(), "Warning:") {
continue
}
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
}
}
})
}
// Name 名称
func (this *Firewalld) Name() string {
return "firewalld"
}
func (this *Firewalld) IsReady() bool {
return this.isReady
}
func (this *Firewalld) AllowPort(port int, protocol string) error {
if !this.isReady {
return nil
}
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
return nil
}
func (this *Firewalld) RemovePort(port int, protocol string) error {
if !this.isReady {
return nil
}
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
return nil
}
func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
}
var args = []string{"--add-rich-rule=rule family='" + family + "' source address='" + ip + "' reject"}
if timeoutSeconds > 0 {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
return nil
}
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
}
var args = []string{"--add-rich-rule=rule family='" + family + "' source address='" + ip + "' drop"}
if timeoutSeconds > 0 {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
return nil
}
func (this *Firewalld) RemoveSourceIP(ip string) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
}
for _, action := range []string{"reject", "drop"} {
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
}
return nil
}
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
select {
case this.cmdQueue <- cmd:
default:
// we discard the command
}
}

View File

@@ -0,0 +1,27 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package firewalls
// FirewallInterface 防火墙接口
type FirewallInterface interface {
// Name 名称
Name() string
// IsReady 是否已准备被调用
IsReady() bool
// AllowPort 允许端口
AllowPort(port int, protocol string) error
// RemovePort 删除端口
RemovePort(port int, protocol string) error
// RejectSourceIP 拒绝某个源IP连接
RejectSourceIP(ip string, timeoutSeconds int) error
// DropSourceIP 丢弃某个源IP数据
DropSourceIP(ip string, timeoutSeconds int) error
// RemoveSourceIP 删除某个源IP
RemoveSourceIP(ip string) error
}

View File

@@ -0,0 +1,55 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package firewalls
// MockFirewall 模拟防火墙
type MockFirewall struct {
}
func NewMockFirewall() *MockFirewall {
return &MockFirewall{}
}
// Name 名称
func (this *MockFirewall) Name() string {
return "mock"
}
// IsReady 是否已准备被调用
func (this *MockFirewall) IsReady() bool {
return true
}
// AllowPort 允许端口
func (this *MockFirewall) AllowPort(port int, protocol string) error {
_ = port
_ = protocol
return nil
}
// RemovePort 删除端口
func (this *MockFirewall) RemovePort(port int, protocol string) error {
_ = port
_ = protocol
return nil
}
// RejectSourceIP 拒绝某个源IP连接
func (this *MockFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
_ = ip
_ = timeoutSeconds
return nil
}
// DropSourceIP 丢弃某个源IP数据
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
_ = ip
_ = timeoutSeconds
return nil
}
// RemoveSourceIP 删除某个源IP
func (this *MockFirewall) RemoveSourceIP(ip string) error {
_ = ip
return nil
}

View File

@@ -71,7 +71,7 @@ func (this *HTTPAPIAction) runAction(action string, listType IPListType, item *p
if err != nil {
return err
}
req.Header.Set("User-Agent", "GoEdge-Node/"+teaconst.Version)
req.Header.Set("User-Agent", teaconst.GlobalProductName+"-Node/"+teaconst.Version)
resp, err := httpAPIClient.Do(req)
if err != nil {
return err

View File

@@ -7,6 +7,7 @@ import (
)
// AllowIP 检查IP是否被允许访问
// 如果一个IP不在任何名单中则允许访问
func AllowIP(ip string, serverId int64) bool {
var ipLong = utils.IP2Long(ip)
if ipLong == 0 {
@@ -40,6 +41,17 @@ func AllowIP(ip string, serverId int64) bool {
return true
}
// IsInWhiteList 检查IP是否在白名单中
func IsInWhiteList(ip string) bool {
var ipLong = utils.IP2Long(ip)
if ipLong == 0 {
return false
}
// check white lists
return GlobalWhiteIPList.Contains(ipLong)
}
// AllowIPStrings 检查一组IP是否被允许访问
func AllowIPStrings(ipStrings []string, serverId int64) bool {
if len(ipStrings) == 0 {

View File

@@ -0,0 +1,156 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCityManager = NewCityManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCityManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCityManager.Stop()
})
}
// CityManager 中国省份信息管理
type CityManager struct {
ticker *time.Ticker
cacheFile string
cityMap map[string]int64 // provinceName_cityName => cityName
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCityManager() *CityManager {
return &CityManager{
cacheFile: Tea.Root + "/configs/region_city.json.cache",
cityMap: map[string]int64{},
}
}
func (this *CityManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("City_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
}
}
func (this *CityManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
this.locker.RLock()
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *CityManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.cityMap = m
}
return nil
}
// 更新城市信息
func (this *CityManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCityRPC().FindAllEnabledRegionCities(rpcClient.Context(), &pb.FindAllEnabledRegionCitiesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, city := range resp.RegionCities {
for _, code := range city.Codes {
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.cityMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -0,0 +1,14 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewCityManager(t *testing.T) {
var manager = NewCityManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup(16, "许昌市"))
}

View File

@@ -9,7 +9,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
@@ -26,16 +25,23 @@ func init() {
SharedCountryManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCountryManager.Stop()
})
}
// CountryManager 国家/地区信息管理
type CountryManager struct {
ticker *time.Ticker
cacheFile string
countryMap map[string]int64 // countryName => countryId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCountryManager() *CountryManager {
@@ -59,11 +65,8 @@ func (this *CountryManager) Start() {
}
// 定时更新
ticker := utils.NewTicker(1 * time.Hour)
events.On(events.EventQuit, func() {
ticker.Stop()
})
for ticker.Next() {
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
@@ -71,6 +74,12 @@ func (this *CountryManager) Start() {
}
}
func (this *CountryManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
this.locker.RLock()
countryId, _ = this.countryMap[countryName]
@@ -101,6 +110,10 @@ func (this *CountryManager) load() error {
// 更新国家信息
func (this *CountryManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
@@ -111,7 +124,7 @@ func (this *CountryManager) loop() error {
}
m := map[string]int64{}
for _, country := range resp.Countries {
for _, country := range resp.RegionCountries {
for _, code := range country.Codes {
m[code] = country.Id
}
@@ -132,6 +145,7 @@ func (this *CountryManager) loop() error {
this.locker.Lock()
this.countryMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存

View File

@@ -23,10 +23,15 @@ func init() {
SharedIPListManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedIPListManager.Stop()
})
}
// IPListManager IP名单管理
type IPListManager struct {
ticker *time.Ticker
db *IPListDB
version int64
@@ -52,17 +57,14 @@ func (this *IPListManager) Start() {
remotelogs.ErrorObject("IP_LIST_MANAGER", err)
}
ticker := time.NewTicker(60 * time.Second)
this.ticker = time.NewTicker(60 * time.Second)
if Tea.IsTesting() {
ticker = time.NewTicker(10 * time.Second)
this.ticker = time.NewTicker(10 * time.Second)
}
events.On(events.EventQuit, func() {
ticker.Stop()
})
countErrors := 0
for {
select {
case <-ticker.C:
case <-this.ticker.C:
case <-IPListUpdateNotify:
}
err := this.loop()
@@ -84,6 +86,12 @@ func (this *IPListManager) Start() {
}
}
func (this *IPListManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *IPListManager) init() {
// 从数据库中当中读取数据
db, err := NewIPListDB()
@@ -197,7 +205,7 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool)
list.Delete(item.Id)
// 从WAF名单中删除
waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId)
waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId, shouldExecute)
// 操作事件
if shouldExecute {

View File

@@ -0,0 +1,155 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedProviderManager = NewProviderManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProviderManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProviderManager.Stop()
})
}
// ProviderManager 中国省份信息管理
type ProviderManager struct {
ticker *time.Ticker
cacheFile string
providerMap map[string]int64 // name => id
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProviderManager() *ProviderManager {
return &ProviderManager{
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
providerMap: map[string]int64{},
}
}
func (this *ProviderManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
}
}
func (this *ProviderManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
this.locker.RLock()
providerId, _ = this.providerMap[providerName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *ProviderManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.providerMap = m
}
return nil
}
// 更新服务商信息
func (this *ProviderManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProviderRPC().FindAllEnabledRegionProviders(rpcClient.Context(), &pb.FindAllEnabledRegionProvidersRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, provider := range resp.RegionProviders {
for _, code := range provider.Codes {
m[code] = provider.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.providerMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -0,0 +1,15 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewProviderManager(t *testing.T) {
var manager = NewProviderManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("阿里云"))
t.Log(manager.Lookup("阿里云2"))
}

View File

@@ -9,7 +9,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
@@ -30,16 +29,23 @@ func init() {
SharedProvinceManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProvinceManager.Stop()
})
}
// ProvinceManager 中国省份信息管理
type ProvinceManager struct {
ticker *time.Ticker
cacheFile string
provinceMap map[string]int64 // provinceName => provinceId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProvinceManager() *ProvinceManager {
@@ -63,11 +69,8 @@ func (this *ProvinceManager) Start() {
}
// 定时更新
ticker := utils.NewTicker(1 * time.Hour)
events.On(events.EventQuit, func() {
ticker.Stop()
})
for ticker.Next() {
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
@@ -75,6 +78,12 @@ func (this *ProvinceManager) Start() {
}
}
func (this *ProvinceManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
this.locker.RLock()
provinceId, _ = this.provinceMap[provinceName]
@@ -103,21 +112,25 @@ func (this *ProvinceManager) load() error {
return nil
}
// 更新国家信息
// 更新省份信息
func (this *ProvinceManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProvinceRPC().FindAllEnabledRegionProvincesWithCountryId(rpcClient.Context(), &pb.FindAllEnabledRegionProvincesWithCountryIdRequest{
CountryId: ChinaCountryId,
RegionCountryId: ChinaCountryId,
})
if err != nil {
return err
}
m := map[string]int64{}
for _, province := range resp.Provinces {
for _, province := range resp.RegionProvinces {
for _, code := range province.Codes {
m[code] = province.Id
}
@@ -138,6 +151,7 @@ func (this *ProvinceManager) loop() error {
this.locker.Lock()
this.provinceMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存

View File

@@ -15,15 +15,22 @@ import (
"time"
)
var SharedUpdater = NewUpdater()
func init() {
events.On(events.EventStart, func() {
updater := NewUpdater()
updater.Start()
goman.New(func() {
SharedUpdater.Start()
})
})
events.On(events.EventQuit, func() {
SharedUpdater.Stop()
})
}
// Updater IP库更新程序
type Updater struct {
ticker *time.Ticker
}
// NewUpdater 获取新对象
@@ -34,15 +41,19 @@ func NewUpdater() *Updater {
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新因为通常不需要更新IP库
ticker := time.NewTicker(1 * time.Hour)
goman.New(func() {
for range ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
this.ticker = time.NewTicker(1 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
})
}
}
func (this *Updater) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
// 单次任务

1
internal/js/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*

View File

@@ -1,50 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"encoding/json"
"github.com/iwind/TeaGo/logs"
"reflect"
)
type Console struct {
}
func (this *Console) Log(args ...interface{}) {
for index, arg := range args {
if arg != nil {
switch arg.(type) {
case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, string:
default:
var argType = reflect.TypeOf(arg)
// 是否有String()方法,如果有直接调用
method, ok := argType.MethodByName("String")
if ok && method.Type.NumIn() == 1 && method.Type.NumOut() == 1 && method.Type.Out(0).Kind() == reflect.String {
args[index] = method.Func.Call([]reflect.Value{reflect.ValueOf(arg)})[0].String()
continue
}
// 转为JSON
argJSON, err := this.toJSON(arg)
if err != nil {
if argType.Kind() == reflect.Func {
args[index] = "[function]"
} else {
args[index] = "[object]"
}
} else {
args[index] = string(argJSON)
}
}
} else {
args[index] = "null"
}
}
logs.Println(append([]interface{}{"[js][console]"}, args...)...)
}
func (this *Console) toJSON(o interface{}) ([]byte, error) {
return json.Marshal(o)
}

View File

@@ -1,38 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"testing"
)
func TestConsole_Log(t *testing.T) {
{
vm := NewVM()
_, err := vm.RunString("console.log('Hello', 'world')")
if err != nil {
t.Fatal(err)
}
}
{
vm := NewVM()
_, err := vm.RunString("console.log(null, true, false, 10, 10.123)")
if err != nil {
t.Fatal(err)
}
}
{
vm := NewVM()
_, err := vm.RunString("console.log({ a:1, b:2 })")
if err != nil {
t.Fatal(err)
}
}
{
vm := NewVM()
_, err := vm.RunString("console.log(console.log)")
if err != nil {
t.Fatal(err)
}
}
}

View File

@@ -1,36 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
type HTTP struct {
r RequestInterface
req *Request
resp *Response
onRequest func(req *Request, resp *Response)
}
func NewHTTP(r RequestInterface) *HTTP {
return &HTTP{
req: NewRequest(r),
resp: NewResponse(r),
}
}
func (this *HTTP) OnRequest(callback func(req *Request, resp *Response)) {
// TODO 考虑是否支持多个callback
this.onRequest = callback
}
func (this *HTTP) OnData(callback func(req *Request, resp *Response)) {
// TODO
}
func (this *HTTP) OnResponse(callback func(req *Request, resp *Response)) {
// TODO
}
func (this *HTTP) TriggerRequest() {
this.onRequest(this.req, this.resp)
}

View File

@@ -1,82 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"bytes"
"io/ioutil"
"net"
)
type Request struct {
r RequestInterface
}
func NewRequest(r RequestInterface) *Request {
return &Request{
r: r,
}
}
func (this *Request) Proto() string {
return this.r.JSRequest().Proto
}
func (this *Request) Method() string {
return this.r.JSRequest().Method
}
func (this *Request) Header() map[string][]string {
return this.r.JSRequest().Header
}
func (this *Request) AddHeader(name string, value string) {
this.r.JSRequest().Header[name] = append(this.r.JSRequest().Header[name], value)
}
func (this *Request) SetHeader(name string, value string) {
this.r.JSRequest().Header[name] = []string{value}
}
func (this *Request) RemoteAddr() string {
var remoteAddr = this.r.JSRequest().RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
return host
}
return remoteAddr
}
func (this *Request) Url() *URL {
return NewURL(this.r.JSRequest().URL)
}
func (this *Request) ContentLength() int64 {
return this.r.JSRequest().ContentLength
}
func (this *Request) Body() []byte {
var bodyReader = this.r.JSRequest().Body
if bodyReader == nil {
return []byte{}
}
data, err := ioutil.ReadAll(bodyReader)
if err != nil {
this.r.JSLog("read body failed: " + err.Error())
}
return data
}
func (this *Request) CopyBody() []byte {
var bodyReader = this.r.JSRequest().Body
if bodyReader == nil {
return []byte{}
}
data, err := ioutil.ReadAll(bodyReader)
if err != nil {
this.r.JSLog("read body failed: " + err.Error())
}
this.r.JSRequest().Body = ioutil.NopCloser(bytes.NewReader(data))
return data
}

View File

@@ -1,19 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import "net/http"
type RequestInterface interface {
// JSRequest 请求
JSRequest() *http.Request
// JSWriter 响应
JSWriter() http.ResponseWriter
// JSStop 中止请求
JSStop()
// JSLog 打印日志
JSLog(msg ...interface{})
}

View File

@@ -1,124 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/js"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"net/http"
"testing"
)
type testRequest struct {
rawRequest *http.Request
rawResponse *testResponse
}
func (this *testRequest) JSRequest() *http.Request {
if this.rawRequest != nil {
return this.rawRequest
}
req, _ := http.NewRequest(http.MethodGet, "https://iwind:123456@goedge.cn/docs?name=Libai&age=20", nil)
req.Header.Set("Server", "edgejs/1.0")
req.Header.Set("Content-Type", "application/json")
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("123456")))
this.rawRequest = req
return req
}
func (this *testRequest) JSWriter() http.ResponseWriter {
if this.rawResponse != nil {
return this.rawResponse
}
this.rawResponse = &testResponse{}
return this.rawResponse
}
func (this *testRequest) JSStop() {
}
func (this *testRequest) JSLog(msg ...interface{}) {
logs.Println(msg...)
}
type testResponse struct {
statusCode int
header http.Header
}
func (this *testResponse) Header() http.Header {
if this.header == nil {
this.header = http.Header{}
}
return this.header
}
func (this *testResponse) Write(p []byte) (int, error) {
return len(p), nil
}
func (this *testResponse) WriteHeader(statusCode int) {
this.statusCode = statusCode
}
func TestRequest(t *testing.T) {
vm := js.NewVM()
vm.SetRequest(&testRequest{})
// 事件监听
_, err := vm.RunString(`
http.onRequest(function (req, resp) {
console.log(req.proto())
let url = req.url()
console.log(url, "port:", url.port(), "args:", url.args())
console.log("username:", url.username(), "password:", url.password())
console.log("uri:", url.uri(), "path:", url.path())
req.addHeader("Server", "1.0")
resp.write("this is response")
console.log(resp)
console.log(req.body())
})
`)
if err != nil {
t.Fatal(err)
}
// 触发事件
_, err = vm.RunString(`http.triggerRequest()`)
if err != nil {
t.Fatal(err)
}
}
func TestRequest_Header(t *testing.T) {
var req = js.NewRequest(&testRequest{})
logs.PrintAsJSON(req.Header(), t)
req.AddHeader("Content-Length", "10")
req.AddHeader("Vary", "1.0")
req.AddHeader("Vary", "2.0")
logs.PrintAsJSON(req.Header(), t)
req.SetHeader("Vary", "3.0")
logs.PrintAsJSON(req.Header(), t)
}
func TestRequest_Body(t *testing.T) {
var req = js.NewRequest(&testRequest{})
t.Log(string(req.Body()))
t.Log(string(req.Body()))
}
func TestRequest_CopyBody(t *testing.T) {
var req = js.NewRequest(&testRequest{})
t.Log(string(req.CopyBody()))
t.Log(string(req.CopyBody()))
}

View File

@@ -1,39 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
type Response struct {
r RequestInterface
}
func NewResponse(r RequestInterface) *Response {
return &Response{
r: r,
}
}
func (this *Response) Write(s string) error {
_, err := this.r.JSWriter().Write([]byte(s))
return err
}
func (this *Response) Reply(status int) {
this.SetStatus(status)
this.r.JSStop()
}
func (this *Response) Header() map[string][]string {
return this.r.JSWriter().Header()
}
func (this *Response) AddHeader(name string, value string) {
this.r.JSWriter().Header()[name] = append(this.r.JSWriter().Header()[name], value)
}
func (this *Response) SetHeader(name string, value string) {
this.r.JSWriter().Header()[name] = []string{value}
}
func (this *Response) SetStatus(statusCode int) {
this.r.JSWriter().WriteHeader(statusCode)
}

View File

@@ -1,16 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js_test
import (
"github.com/TeaOSLab/EdgeNode/internal/js"
"testing"
)
func TestNewResponse(t *testing.T) {
var resp = js.NewResponse(&testRequest{})
resp.AddHeader("Vary", "1.0")
resp.AddHeader("Vary", "2.0")
resp.SetHeader("Server", "edgejs/1.0")
t.Logf("%#v", resp.Header())
}

View File

@@ -1,90 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"github.com/dop251/goja"
"github.com/iwind/TeaGo/types"
"net/url"
)
type URL struct {
u *url.URL
}
func NewURL(u *url.URL) *URL {
return &URL{
u: u,
}
}
func (this *URL) JSNew(args []goja.Value) *URL {
var urlString = ""
if len(args) == 1 {
urlString = args[0].String()
}
u, _ := url.Parse(urlString)
if u == nil {
u = &url.URL{}
}
return NewURL(u)
}
func (this *URL) Port() int {
return types.Int(this.u.Port())
}
func (this *URL) Args() map[string][]string {
return this.u.Query()
}
func (this *URL) Arg(name string) string {
return this.u.Query().Get(name)
}
func (this *URL) Username() string {
if this.u.User != nil {
return this.u.User.Username()
}
return ""
}
func (this *URL) Password() string {
if this.u.User != nil {
password, _ := this.u.User.Password()
return password
}
return ""
}
func (this *URL) Uri() string {
return this.u.RequestURI()
}
func (this *URL) Path() string {
return this.u.Path
}
func (this *URL) Host() string {
return this.u.Host
}
func (this *URL) Fragment() string {
return this.u.Fragment
}
func (this *URL) Hash() string {
if len(this.u.Fragment) > 0 {
return "#" + this.u.Fragment
} else {
return ""
}
}
func (this *URL) Scheme() string {
return this.u.Scheme
}
func (this *URL) String() string {
return this.u.String()
}

View File

@@ -1,18 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"net/url"
"testing"
)
func TestURL(t *testing.T) {
raw, err := url.Parse("https://iwind:123456@goedge.cn/docs?name=Libai&age=20#a=b")
if err != nil {
t.Fatal(err)
}
var u = NewURL(raw)
t.Log("host:", u.Host())
t.Log("hash:", u.Hash())
}

View File

@@ -1,153 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"errors"
"github.com/dop251/goja"
"github.com/iwind/TeaGo/logs"
"reflect"
"strings"
)
var sharedPrograms []*goja.Program
var sharedConsole = &Console{}
func init() {
// compile programs
}
type VM struct {
vm *goja.Runtime
}
func NewVM() *VM {
vm := goja.New()
vm.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
// programs
for _, program := range sharedPrograms {
_, _ = vm.RunProgram(program)
}
v := &VM{vm: vm}
v.initVM()
return v
}
func (this *VM) Set(name string, obj interface{}) error {
return this.vm.Set(name, obj)
}
func (this *VM) AddConstructor(name string, instance interface{}) error {
objType := reflect.TypeOf(instance)
if objType.Kind() != reflect.Ptr {
return errors.New("instance should be pointer")
}
// construct
newMethod, ok := objType.MethodByName("JSNew")
if !ok {
return errors.New("can not find 'JSNew()' method in '" + objType.Elem().Name() + "'")
}
var err = this.Set(name, func(call goja.ConstructorCall) *goja.Object {
if newMethod.Type.NumIn() != 2 {
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should accept a '[]goja.Value' argument"))
return nil
}
if newMethod.Type.In(1).String() != "[]goja.Value" {
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should accept a '[]goja.Value' argument"))
return nil
}
// new
var results = newMethod.Func.Call([]reflect.Value{reflect.ValueOf(instance), reflect.ValueOf(call.Arguments)})
if len(results) == 0 {
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should return a valid instance"))
return nil
}
var result = results[0]
if result.Type() != objType {
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should return a same instance"))
return nil
}
// methods
var resultType = result.Type()
var numMethod = result.NumMethod()
for i := 0; i < numMethod; i++ {
var method = resultType.Method(i)
var methodName = strings.ToLower(method.Name[:1]) + method.Name[1:]
err := call.This.Set(methodName, result.MethodByName(method.Name).Interface())
if err != nil {
this.throw(err)
continue
}
}
// 支持属性
var numField = result.Elem().Type().NumField()
for i := 0; i < numField; i++ {
var field = result.Elem().Field(i)
if !field.CanInterface() {
continue
}
var fieldType = objType.Elem().Field(i)
tag, ok := fieldType.Tag.Lookup("json")
if !ok {
tag = fieldType.Name
tag = strings.ToLower(tag[:1]) + tag[1:]
} else {
// TODO 校验tag是否符合变量语法
}
err := call.This.Set(tag, field.Interface())
if err != nil {
this.throw(err)
continue
}
}
return nil
})
return err
}
func (this *VM) RunString(str string) (goja.Value, error) {
defer func() {
e := recover()
if e != nil {
// TODO 需要打印trace
logs.Println("panic:", e)
}
}()
return this.vm.RunString(str)
}
func (this *VM) SetRequest(req RequestInterface) {
{
err := this.vm.Set("http", NewHTTP(req))
if err != nil {
this.throw(err)
}
}
}
func (this *VM) initVM() {
{
err := this.vm.Set("console", sharedConsole)
if err != nil {
this.throw(err)
}
}
}
func (this *VM) throw(err error) {
if err == nil {
return
}
// TODO
logs.Println("js:VM:error: " + err.Error())
}

View File

@@ -1,158 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package js
import (
"github.com/dop251/goja"
"testing"
"time"
)
func TestNewVM(t *testing.T) {
before := time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
vm := NewVM()
{
v, err := vm.RunString("JSON.stringify({\"a\":\"b\"})")
if err != nil {
t.Fatal(err)
}
t.Log("JSON.stringify():", v)
}
{
v, err := vm.RunString(`JSON.parse('{\"a\":\"b\"}')`)
if err != nil {
t.Fatal(err)
}
t.Log("JSON.parse():", v)
}
{
err := vm.AddConstructor("Url", &URL{})
if err != nil {
t.Fatal("add constructor error:", err)
}
_, err = vm.RunString(`
{
let u = new Url("https://goedge.cn/docs?v=1")
console.log("host:", u.host(), u.uri())
}
{
let u = new Url("https://teaos.cn/downloads?v=1")
console.log("host:", u.host(), u.uri())
}
{
let u = new Url()
console.log("host:", u.host(), u.uri())
}
{
let u = new Url("a", "b", "c")
console.log("host:", u.host(), u.uri())
}
`)
if err != nil {
t.Fatal("add constructor error:" + err.Error())
}
}
}
func TestVM_Program(t *testing.T) {
var s = `
{
let u = new Url("https://goedge.cn/docs?v=1")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("https://teaos.cn/downloads?v=1")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url()
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("a", "b", "c")
//console.log("host:", u.host(), u.uri())
}
`
program := goja.MustCompile("s", s, true)
before := time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
vm := NewVM()
err := vm.AddConstructor("Url", &URL{})
if err != nil {
t.Fatal("add constructor error:", err)
}
//_, err = vm.RunString(s)
_, err = vm.vm.RunProgram(program)
if err != nil {
t.Fatal("add constructor error:" + err.Error())
}
}
func Benchmark_Program(b *testing.B) {
var s = `
{
let u = new Url("https://goedge.cn/docs?v=1")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("https://teaos.cn/downloads?v=1")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url()
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("a", "b", "c")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("https://goedge.cn/docs?v=1")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("https://teaos.cn/downloads?v=1")
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url()
//console.log("host:", u.host(), u.uri())
}
{
let u = new Url("a", "b", "c")
//console.log("host:", u.host(), u.uri())
}
`
program := goja.MustCompile("s", s, true)
vm := NewVM()
err := vm.AddConstructor("Url", &URL{})
if err != nil {
b.Fatal("add constructor error:", err)
}
for i := 0; i < b.N; i++ {
//_, err = vm.RunString(s)
_, err = vm.vm.RunProgram(program)
if err != nil {
b.Fatal("add constructor error:" + err.Error())
}
}
}

View File

@@ -9,6 +9,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/configs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/errors"
@@ -41,7 +42,7 @@ func NewAPIStream() *APIStream {
}
func (this *APIStream) Start() {
events.On(events.EventQuit, func() {
events.OnKey(events.EventQuit, this, func() {
this.isQuiting = true
if this.cancelFunc != nil {
this.cancelFunc()
@@ -182,7 +183,7 @@ func (this *APIStream) handleWriteCache(message *pb.NodeStreamMessage) error {
}
expiredAt := time.Now().Unix() + msg.LifeSeconds
writer, err := storage.OpenWriter(msg.Key, expiredAt, 200)
writer, err := storage.OpenWriter(msg.Key, expiredAt, 200, int64(len(msg.Value)), false)
if err != nil {
this.replyFail(message.RequestId, "prepare writing failed: "+err.Error())
return err
@@ -350,7 +351,12 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
if msg.Type == "file" {
var keys = msg.Keys
for _, key := range keys {
keys = append(keys, key+webpSuffix)
keys = append(keys, key+webpCacheSuffix, key+cacheMethodSuffix+"HEAD")
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
keys = append(keys, key+compressionCacheSuffix+encoding)
keys = append(keys, key+webpCacheSuffix+compressionCacheSuffix+encoding)
}
}
msg.Keys = keys
}
@@ -462,7 +468,7 @@ func (this *APIStream) handlePreheatCache(message *pb.NodeStreamMessage) error {
}
expiredAt := time.Now().Unix() + 8600
writer, err := storage.OpenWriter(key, expiredAt, 200) // TODO 可以设置缓存过期时间
writer, err := storage.OpenWriter(key, expiredAt, 200, resp.ContentLength, false) // TODO 可以设置缓存过期时间
if err != nil {
locker.Lock()
errorMessages = append(errorMessages, "open cache writer failed: "+key+": "+err.Error())

View File

@@ -4,9 +4,16 @@ package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/ratelimit"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/types"
"net"
"os"
"sync"
"sync/atomic"
"time"
@@ -17,8 +24,11 @@ type ClientConn struct {
once sync.Once
globalLimiter *ratelimit.Counter
isTLS bool
hasRead bool
isTLS bool
hasDeadline bool
hasRead bool
hasResetSYNFlood bool
BaseClientConn
}
@@ -38,9 +48,9 @@ func NewClientConn(conn net.Conn, isTLS bool, quickClose bool, globalLimiter *ra
func (this *ClientConn) Read(b []byte) (n int, err error) {
if this.isTLS {
if !this.hasRead {
_ = this.rawConn.SetReadDeadline(time.Now().Add(5 * time.Second)) // TODO 握手超时时间可以设置
this.hasRead = true
if !this.hasDeadline {
_ = this.rawConn.SetReadDeadline(time.Now().Add(time.Duration(nodeconfigs.DefaultTLSHandshakeTimeout) * time.Second)) // TODO 握手超时时间可以设置
this.hasDeadline = true
defer func() {
_ = this.rawConn.SetReadDeadline(time.Time{})
}()
@@ -48,10 +58,28 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
}
n, err = this.rawConn.Read(b)
if n > 0 {
atomic.AddUint64(&teaconst.InTrafficBytes, uint64(n))
if !this.hasRead {
this.hasRead = true
}
}
// SYN Flood检测
var isHandshakeError = err != nil && os.IsTimeout(err) && !this.hasRead
if isHandshakeError {
_ = this.SetLinger(0)
}
var synFloodConfig = sharedNodeConfig.SYNFloodConfig()
if synFloodConfig != nil && synFloodConfig.IsOn {
if isHandshakeError {
this.increaseSYNFlood(synFloodConfig)
} else if err == nil && !this.hasResetSYNFlood {
this.hasResetSYNFlood = true
this.resetSYNFlood()
}
}
return
}
@@ -66,6 +94,8 @@ func (this *ClientConn) Write(b []byte) (n int, err error) {
func (this *ClientConn) Close() error {
this.isClosed = true
err := this.rawConn.Close()
// 全局并发数限制
this.once.Do(func() {
if this.globalLimiter != nil {
@@ -76,7 +106,7 @@ func (this *ClientConn) Close() error {
// 单个服务并发数限制
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
return this.rawConn.Close()
return err
}
func (this *ClientConn) LocalAddr() net.Addr {
@@ -98,3 +128,30 @@ func (this *ClientConn) SetReadDeadline(t time.Time) error {
func (this *ClientConn) SetWriteDeadline(t time.Time) error {
return this.rawConn.SetWriteDeadline(t)
}
func (this *ClientConn) resetSYNFlood() {
ttlcache.SharedCache.Delete("SYN_FLOOD:" + this.RawIP())
}
func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloodConfig) {
var ip = this.RawIP()
if len(ip) > 0 && !iplibrary.IsInWhiteList(ip) && (!synFloodConfig.IgnoreLocal || !utils.IsLocalIP(ip)) {
var timestamp = utils.NextMinuteUnixTime()
var result = ttlcache.SharedCache.IncreaseInt64("SYN_FLOOD:"+ip, 1, timestamp)
var minAttempts = synFloodConfig.MinAttempts
if minAttempts < 5 {
minAttempts = 5
}
if !this.isTLS {
// 非TLS设置为两倍防止误封
minAttempts = 2 * minAttempts
}
if result >= int64(minAttempts) {
var timeout = synFloodConfig.TimeoutSeconds
if timeout <= 0 {
timeout = 600
}
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击当前1分钟"+types.String(result)+"次空连接")
}
}
}

View File

@@ -36,3 +36,23 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
return sharedClientConnLimiter.Add(this.rawConn.RemoteAddr().String(), serverId, remoteAddr, maxConnsPerServer, maxConnsPerIP)
}
// RawIP 原本IP
func (this *BaseClientConn) RawIP() string {
ip, _, _ := net.SplitHostPort(this.rawConn.RemoteAddr().String())
return ip
}
// TCPConn 转换为TCPConn
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
conn, ok := this.rawConn.(*net.TCPConn)
return conn, ok
}
// SetLinger 设置Linger
func (this *BaseClientConn) SetLinger(seconds int) error {
tcpConn, ok := this.TCPConn()
if ok {
return tcpConn.SetLinger(seconds)
}
return nil
}

View File

@@ -16,7 +16,7 @@ import (
// 发送监控流量
func init() {
events.On(events.EventStart, func() {
ticker := time.NewTicker(1 * time.Minute)
var ticker = time.NewTicker(1 * time.Minute)
goman.New(func() {
for range ticker.C {
// 加入到数据队列中

View File

@@ -20,7 +20,7 @@ type ClientListener struct {
quickClose bool
}
func NewClientListener1(listener net.Listener, quickClose bool) *ClientListener {
func NewClientListener(listener net.Listener, quickClose bool) *ClientListener {
return &ClientListener{
rawListener: listener,
quickClose: quickClose,

View File

@@ -6,6 +6,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"strings"
"time"
)
@@ -108,37 +109,33 @@ Loop:
}
func (this *HTTPAccessLogQueue) toValidUTF8(accessLog *pb.HTTPAccessLog) {
accessLog.RemoteUser = this.toValidUTF8string(accessLog.RemoteUser)
accessLog.RequestURI = this.toValidUTF8string(accessLog.RequestURI)
accessLog.RequestPath = this.toValidUTF8string(accessLog.RequestPath)
accessLog.RequestFilename = this.toValidUTF8string(accessLog.RequestFilename)
accessLog.RemoteUser = utils.ToValidUTF8string(accessLog.RemoteUser)
accessLog.RequestURI = utils.ToValidUTF8string(accessLog.RequestURI)
accessLog.RequestPath = utils.ToValidUTF8string(accessLog.RequestPath)
accessLog.RequestFilename = utils.ToValidUTF8string(accessLog.RequestFilename)
accessLog.RequestBody = bytes.ToValidUTF8(accessLog.RequestBody, []byte{})
for _, v := range accessLog.SentHeader {
for index, s := range v.Values {
v.Values[index] = this.toValidUTF8string(s)
v.Values[index] = utils.ToValidUTF8string(s)
}
}
accessLog.Referer = this.toValidUTF8string(accessLog.Referer)
accessLog.UserAgent = this.toValidUTF8string(accessLog.UserAgent)
accessLog.Request = this.toValidUTF8string(accessLog.Request)
accessLog.ContentType = this.toValidUTF8string(accessLog.ContentType)
accessLog.Referer = utils.ToValidUTF8string(accessLog.Referer)
accessLog.UserAgent = utils.ToValidUTF8string(accessLog.UserAgent)
accessLog.Request = utils.ToValidUTF8string(accessLog.Request)
accessLog.ContentType = utils.ToValidUTF8string(accessLog.ContentType)
for k, c := range accessLog.Cookie {
accessLog.Cookie[k] = this.toValidUTF8string(c)
accessLog.Cookie[k] = utils.ToValidUTF8string(c)
}
accessLog.Args = this.toValidUTF8string(accessLog.Args)
accessLog.QueryString = this.toValidUTF8string(accessLog.QueryString)
accessLog.Args = utils.ToValidUTF8string(accessLog.Args)
accessLog.QueryString = utils.ToValidUTF8string(accessLog.QueryString)
for _, v := range accessLog.Header {
for index, s := range v.Values {
v.Values[index] = this.toValidUTF8string(s)
v.Values[index] = utils.ToValidUTF8string(s)
}
}
}
func (this *HTTPAccessLogQueue) toValidUTF8string(v string) string {
return strings.ToValidUTF8(v, "")
}

View File

@@ -84,14 +84,13 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
if idleConns <= 0 {
idleConns = numberCPU * 8
}
//logs.Println("[ORIGIN]max connections:", maxConnections)
// TLS通讯
tlsConfig := &tls.Config{
var tlsConfig = &tls.Config{
InsecureSkipVerify: true,
}
if origin.Cert != nil {
obj := origin.Cert.CertObject()
var obj = origin.Cert.CertObject()
if obj != nil {
tlsConfig.InsecureSkipVerify = false
tlsConfig.Certificates = []tls.Certificate{*obj}
@@ -101,37 +100,16 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
}
}
transport := &http.Transport{
var transport = &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
// 支持TOA的连接
toaConfig := sharedTOAManager.Config()
if toaConfig != nil && toaConfig.IsOn {
retries := 3
for i := 1; i <= retries; i++ {
port := int(toaConfig.RandLocalPort())
// TODO 思考是否支持X-Real-IP/X-Forwarded-IP
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + req.requestRemoteAddr(true))
if err != nil {
remotelogs.Error("TOA", "add failed: "+err.Error())
} else {
dialer := net.Dialer{
Timeout: connectionTimeout,
KeepAlive: 1 * time.Minute,
LocalAddr: &net.TCPAddr{
Port: port,
},
}
conn, err := dialer.DialContext(ctx, network, originAddr)
// TODO 需要在合适的时机删除TOA记录
if err == nil || i == retries {
return conn, err
}
}
}
conn, err := this.handleTOA(req, ctx, network, originAddr, connectionTimeout)
if conn != nil || err != nil {
return conn, err
}
// 普通的连接
conn, err := (&net.Dialer{
conn, err = (&net.Dialer{
Timeout: connectionTimeout,
KeepAlive: 1 * time.Minute,
}).DialContext(ctx, network, originAddr)
@@ -139,32 +117,10 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
return nil, err
}
if proxyProtocol != nil && proxyProtocol.IsOn && (proxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || proxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
var remoteAddr = req.requestRemoteAddr(true)
var transportProtocol = proxyproto.TCPv4
if strings.Contains(remoteAddr, ":") {
transportProtocol = proxyproto.TCPv6
}
var destAddr = conn.RemoteAddr()
var reqConn = req.RawReq.Context().Value(HTTPConnContextKey)
if reqConn != nil {
destAddr = reqConn.(net.Conn).LocalAddr()
}
header := proxyproto.Header{
Version: byte(proxyProtocol.Version),
Command: proxyproto.PROXY,
TransportProtocol: transportProtocol,
SourceAddr: &net.TCPAddr{
IP: net.ParseIP(remoteAddr),
Port: req.requestRemotePort(),
},
DestinationAddr: destAddr,
}
_, err = header.WriteTo(conn)
if err != nil {
_ = conn.Close()
return nil, err
}
// 处理PROXY protocol
err = this.handlePROXYProtocol(conn, req, proxyProtocol)
if err != nil {
return nil, err
}
return conn, nil
@@ -174,7 +130,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
MaxConnsPerHost: maxConnections,
IdleConnTimeout: idleTimeout,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
TLSClientConfig: tlsConfig,
Proxy: nil,
}
@@ -208,3 +164,69 @@ func (this *HTTPClientPool) cleanClients() {
this.locker.Unlock()
}
}
// 支持TOA
func (this *HTTPClientPool) handleTOA(req *HTTPRequest, ctx context.Context, network string, originAddr string, connectionTimeout time.Duration) (net.Conn, error) {
// TODO 每个服务读取自身所属集群的TOA设置
toaConfig := sharedTOAManager.Config()
if toaConfig != nil && toaConfig.IsOn {
retries := 3
for i := 1; i <= retries; i++ {
port := int(toaConfig.RandLocalPort())
// TODO 思考是否支持X-Real-IP/X-Forwarded-IP
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + req.requestRemoteAddr(true))
if err != nil {
remotelogs.Error("TOA", "add failed: "+err.Error())
} else {
dialer := net.Dialer{
Timeout: connectionTimeout,
KeepAlive: 1 * time.Minute,
LocalAddr: &net.TCPAddr{
Port: port,
},
}
conn, err := dialer.DialContext(ctx, network, originAddr)
// TODO 需要在合适的时机删除TOA记录
if err == nil || i == retries {
return conn, err
}
}
}
}
return nil, nil
}
// 支持PROXY Protocol
func (this *HTTPClientPool) handlePROXYProtocol(conn net.Conn, req *HTTPRequest, proxyProtocol *serverconfigs.ProxyProtocolConfig) error {
if proxyProtocol != nil && proxyProtocol.IsOn && (proxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || proxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
var remoteAddr = req.requestRemoteAddr(true)
var transportProtocol = proxyproto.TCPv4
if strings.Contains(remoteAddr, ":") {
transportProtocol = proxyproto.TCPv6
}
var destAddr = conn.RemoteAddr()
var reqConn = req.RawReq.Context().Value(HTTPConnContextKey)
if reqConn != nil {
destAddr = reqConn.(net.Conn).LocalAddr()
}
header := proxyproto.Header{
Version: byte(proxyProtocol.Version),
Command: proxyproto.PROXY,
TransportProtocol: transportProtocol,
SourceAddr: &net.TCPAddr{
IP: net.ParseIP(remoteAddr),
Port: req.requestRemotePort(),
},
DestinationAddr: destAddr,
}
_, err := header.WriteTo(conn)
if err != nil {
_ = conn.Close()
return err
}
return nil
}
return nil
}

View File

@@ -8,10 +8,12 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"io"
"io/ioutil"
@@ -38,8 +40,8 @@ type HTTPRequest struct {
// 外部参数
RawReq *http.Request
RawWriter http.ResponseWriter
Server *serverconfigs.ServerConfig
Host string // 请求的Host
ReqServer *serverconfigs.ServerConfig
ReqHost string // 请求的Host
ServerName string // 实际匹配到的Host
ServerAddr string // 实际启动的服务器监听地址
IsHTTP bool
@@ -63,6 +65,7 @@ type HTTPRequest struct {
rewriteRule *serverconfigs.HTTPRewriteRule // 匹配到的重写规则
rewriteReplace string // 重写规则的目标
rewriteIsExternalURL bool // 重写目标是否为外部URL
remoteAddr string // 计算后的RemoteAddr
cacheRef *serverconfigs.HTTPCacheRef // 缓存设置
cacheKey string // 缓存使用的Key
@@ -83,6 +86,9 @@ type HTTPRequest struct {
logAttrs map[string]string
disableLog bool // 此请求中关闭Log
// script相关操作
isDone bool
}
// 初始化
@@ -95,7 +101,7 @@ func (this *HTTPRequest) init() {
// this.uri = this.RawReq.URL.RequestURI()
// 之所以不使用RequestURI()是不想让URL中的Path被Encode
var urlPath = this.RawReq.URL.Path
if this.Server.Web != nil && this.Server.Web.MergeSlashes {
if this.ReqServer.Web != nil && this.ReqServer.Web.MergeSlashes {
urlPath = utils.CleanPath(urlPath)
this.web.MergeSlashes = true
}
@@ -126,19 +132,26 @@ func (this *HTTPRequest) Do() {
this.init()
// 当前服务的反向代理配置
if this.Server.ReverseProxyRef != nil && this.Server.ReverseProxy != nil {
this.reverseProxyRef = this.Server.ReverseProxyRef
this.reverseProxy = this.Server.ReverseProxy
if this.ReqServer.ReverseProxyRef != nil && this.ReqServer.ReverseProxy != nil {
this.reverseProxyRef = this.ReqServer.ReverseProxyRef
this.reverseProxy = this.ReqServer.ReverseProxy
}
// Web配置
err := this.configureWeb(this.Server.Web, true, 0)
err := this.configureWeb(this.ReqServer.Web, true, 0)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.doEnd()
return
}
// 回调事件
this.onInit()
if this.writer.isFinished {
this.doEnd()
return
}
// 特殊URL处理
if len(this.rawURI) > 1 && this.rawURI[1] == '.' {
// ACME
@@ -151,14 +164,14 @@ func (this *HTTPRequest) Do() {
}
// 套餐
if this.Server.UserPlan != nil && !this.Server.UserPlan.IsAvailable() {
if this.ReqServer.UserPlan != nil && !this.ReqServer.UserPlan.IsAvailable() {
this.doPlanExpires()
this.doEnd()
return
}
// 流量限制
if this.Server.TrafficLimit != nil && this.Server.TrafficLimit.IsOn && !this.Server.TrafficLimit.IsEmpty() && this.Server.TrafficLimitStatus != nil && this.Server.TrafficLimitStatus.IsValid() {
if this.ReqServer.TrafficLimit != nil && this.ReqServer.TrafficLimit.IsOn && !this.ReqServer.TrafficLimit.IsEmpty() && this.ReqServer.TrafficLimitStatus != nil && this.ReqServer.TrafficLimitStatus.IsValid() {
this.doTrafficLimit()
this.doEnd()
return
@@ -300,14 +313,14 @@ func (this *HTTPRequest) doEnd() {
// 流量统计
// TODO 增加是否开启开关
// TODO 增加Header统计考虑从Conn中读取
if this.Server != nil {
if this.ReqServer != nil {
if this.isCached {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, this.writer.sentBodyBytes, 1, 1, 0, 0, this.Server.ShouldCheckTrafficLimit(), this.Server.PlanId())
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes(), this.writer.SentBodyBytes(), 1, 1, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
} else {
if this.isAttack {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 1, this.writer.sentBodyBytes, this.Server.ShouldCheckTrafficLimit(), this.Server.PlanId())
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes(), 0, 1, 0, 1, this.writer.SentBodyBytes(), this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
} else {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 0, 0, this.Server.ShouldCheckTrafficLimit(), this.Server.PlanId())
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes(), 0, 1, 0, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
}
}
}
@@ -452,6 +465,26 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
this.web.RequestLimit = web.RequestLimit
}
// request scripts
if web.RequestScripts != nil {
if this.web.RequestScripts == nil {
this.web.RequestScripts = web.RequestScripts
} else {
if web.RequestScripts.InitGroup != nil && (web.RequestScripts.InitGroup.IsPrior || isTop) {
if this.web.RequestScripts == nil {
this.web.RequestScripts = &serverconfigs.HTTPRequestScriptsConfig{}
}
this.web.RequestScripts.InitGroup = web.RequestScripts.InitGroup
}
if web.RequestScripts.RequestGroup != nil && (web.RequestScripts.RequestGroup.IsPrior || isTop) {
if this.web.RequestScripts == nil {
this.web.RequestScripts = &serverconfigs.HTTPRequestScriptsConfig{}
}
this.web.RequestScripts.RequestGroup = web.RequestScripts.RequestGroup
}
}
}
// 重写规则
if len(web.RewriteRefs) > 0 {
for index, ref := range web.RewriteRefs {
@@ -521,7 +554,7 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
}
if varMapping, isMatched := location.Match(rawPath, this.Format); isMatched {
// 检查专属域名
if len(location.Domains) > 0 && !configutils.MatchDomains(location.Domains, this.Host) {
if len(location.Domains) > 0 && !configutils.MatchDomains(location.Domains, this.ReqHost) {
continue
}
@@ -603,11 +636,11 @@ func (this *HTTPRequest) Format(source string) string {
if this.IsHTTPS {
scheme = "https"
}
return scheme + "://" + this.Host + this.rawURI
return scheme + "://" + this.ReqHost + this.rawURI
case "requestPath":
return this.requestPath()
return this.Path()
case "requestPathExtension":
return filepath.Ext(this.requestPath())
return filepath.Ext(this.Path())
case "requestLength":
return strconv.FormatInt(this.requestLength(), 10)
case "requestTime":
@@ -621,7 +654,7 @@ func (this *HTTPRequest) Format(source string) string {
}
if this.web.Root != nil && this.web.Root.IsOn {
return filepath.Clean(this.web.Root.Dir + this.requestPath())
return filepath.Clean(this.web.Root.Dir + this.Path())
}
return ""
@@ -650,7 +683,7 @@ func (this *HTTPRequest) Format(source string) string {
case "timestamp":
return strconv.FormatInt(this.requestFromTime.Unix(), 10)
case "host":
return this.Host
return this.ReqHost
case "referer":
return this.RawReq.Referer()
case "referer.host":
@@ -768,7 +801,7 @@ func (this *HTTPRequest) Format(source string) string {
// host
if prefix == "host" {
pieces := strings.Split(this.Host, ".")
pieces := strings.Split(this.ReqHost, ".")
switch suffix {
case "first":
if len(pieces) > 0 {
@@ -833,6 +866,104 @@ func (this *HTTPRequest) Format(source string) string {
}
}
// geo
if prefix == "geo" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
switch suffix {
case "country.name":
if result != nil {
return result.Country
}
return ""
case "country.id":
if result != nil {
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
}
return "0"
case "province.name":
if result != nil {
return result.Province
}
return ""
case "province.id":
if result != nil {
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
}
return "0"
case "city.name":
if result != nil {
return result.City
}
return ""
case "city.id":
if result != nil {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 {
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
} else {
return "0"
}
}
return "0"
}
}
// ips
if prefix == "isp" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
switch suffix {
case "name":
if result != nil {
return result.ISP
}
case "id":
if result != nil {
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
}
return "0"
}
return ""
}
// browser
if prefix == "browser" {
var result = stats.SharedUserAgentParser.Parse(this.RawReq.UserAgent())
switch suffix {
case "os.name":
return result.OS.Name
case "os.version":
return result.OS.Version
case "name":
return result.BrowserName
case "version":
return result.BrowserVersion
case "isMobile":
if result.IsMobile {
return "1"
} else {
return "0"
}
}
}
// product
if prefix == "product" {
switch suffix {
case "name":
if sharedNodeConfig.ProductConfig != nil && len(sharedNodeConfig.ProductConfig.Name) > 0 {
return sharedNodeConfig.ProductConfig.Name
}
return teaconst.GlobalProductName
case "version":
if sharedNodeConfig.ProductConfig != nil && len(sharedNodeConfig.ProductConfig.Version) > 0 {
return sharedNodeConfig.ProductConfig.Version
}
return teaconst.Version
}
}
return "${" + varName + "}"
})
}
@@ -846,12 +977,17 @@ func (this *HTTPRequest) addVarMapping(varMapping map[string]string) {
// 获取请求的客户端地址
func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
if supportVar && len(this.remoteAddr) > 0 {
return this.remoteAddr
}
if supportVar &&
this.web.RemoteAddr != nil &&
this.web.RemoteAddr.IsOn &&
!this.web.RemoteAddr.IsEmpty() {
var remoteAddr = this.Format(this.web.RemoteAddr.Value)
if net.ParseIP(remoteAddr) != nil {
this.remoteAddr = remoteAddr
return remoteAddr
}
}
@@ -864,6 +1000,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
forwardedFor = forwardedFor[:commaIndex]
}
if net.ParseIP(forwardedFor) != nil {
if supportVar {
this.remoteAddr = forwardedFor
}
return forwardedFor
}
}
@@ -873,6 +1012,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
realIP, ok := this.RawReq.Header["X-Real-IP"]
if ok && len(realIP) > 0 {
if net.ParseIP(realIP[0]) != nil {
if supportVar {
this.remoteAddr = realIP[0]
}
return realIP[0]
}
}
@@ -883,6 +1025,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
realIP, ok := this.RawReq.Header["X-Real-Ip"]
if ok && len(realIP) > 0 {
if net.ParseIP(realIP[0]) != nil {
if supportVar {
this.remoteAddr = realIP[0]
}
return realIP[0]
}
}
@@ -892,6 +1037,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
remoteAddr := this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
if supportVar {
this.remoteAddr = host
}
return host
} else {
return remoteAddr
@@ -950,8 +1098,8 @@ func (this *HTTPRequest) requestRemoteUser() string {
return username
}
// 请求的URL中路径部分
func (this *HTTPRequest) requestPath() string {
// Path 请求的URL中路径部分
func (this *HTTPRequest) Path() string {
uri, err := url.ParseRequestURI(this.rawURI)
if err != nil {
return ""
@@ -1061,9 +1209,148 @@ func (this *HTTPRequest) requestServerPort() int {
return 0
}
// 获取完整的URL
func (this *HTTPRequest) requestFullURL() string {
return this.requestScheme() + "://" + this.Host + this.uri
func (this *HTTPRequest) Id() string {
return this.requestId
}
func (this *HTTPRequest) Server() maps.Map {
return maps.Map{"id": this.ReqServer.Id}
}
func (this *HTTPRequest) Node() maps.Map {
return maps.Map{"id": teaconst.NodeId}
}
// URL 获取完整的URL
func (this *HTTPRequest) URL() string {
return this.requestScheme() + "://" + this.ReqHost + this.uri
}
// Host 获取Host
func (this *HTTPRequest) Host() string {
return this.ReqHost
}
func (this *HTTPRequest) Proto() string {
return this.RawReq.Proto
}
func (this *HTTPRequest) ProtoMajor() int {
return this.RawReq.ProtoMajor
}
func (this *HTTPRequest) ProtoMinor() int {
return this.RawReq.ProtoMinor
}
func (this *HTTPRequest) RemoteAddr() string {
return this.requestRemoteAddr(true)
}
func (this *HTTPRequest) RawRemoteAddr() string {
addr := this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
}
return addr
}
func (this *HTTPRequest) RemotePort() int {
addr := this.RawReq.RemoteAddr
_, port, err := net.SplitHostPort(addr)
if err != nil {
return 0
}
return types.Int(port)
}
func (this *HTTPRequest) SetAttr(name string, value string) {
this.logAttrs[name] = value
}
func (this *HTTPRequest) SetVar(name string, value string) {
this.varMapping[name] = value
}
// ContentLength 请求内容长度
func (this *HTTPRequest) ContentLength() int64 {
return this.RawReq.ContentLength
}
// Method 请求方法
func (this *HTTPRequest) Method() string {
return this.RawReq.Method
}
// TransferEncoding 获取传输编码
func (this *HTTPRequest) TransferEncoding() string {
if len(this.RawReq.TransferEncoding) > 0 {
return this.RawReq.TransferEncoding[0]
}
return ""
}
// Cookie 获取Cookie
func (this *HTTPRequest) Cookie(name string) string {
c, err := this.RawReq.Cookie(name)
if err != nil {
return ""
}
return c.Value
}
// DeleteHeader 删除Header
func (this *HTTPRequest) DeleteHeader(name string) {
this.RawReq.Header.Del(name)
}
// SetHeader 设置Header
func (this *HTTPRequest) SetHeader(name string, values []string) {
this.RawReq.Header[name] = values
}
// Header 读取Header
func (this *HTTPRequest) Header() http.Header {
return this.RawReq.Header
}
// URI 获取当前请求的URI
func (this *HTTPRequest) URI() string {
return this.uri
}
// SetURI 设置当前请求的URI
func (this *HTTPRequest) SetURI(uri string) {
this.uri = uri
}
// Done 设置已完成
func (this *HTTPRequest) Done() {
this.isDone = true
}
// Close 关闭连接
func (this *HTTPRequest) Close() {
this.Done()
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
if requestConn == nil {
return
}
conn, ok := requestConn.(net.Conn)
if ok {
_ = conn.Close()
return
}
return
}
// Allow 放行
func (this *HTTPRequest) Allow() {
this.web.FirewallRef = nil
}
// 设置代理相关头部信息
@@ -1106,9 +1393,9 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
/**{
forwarded, ok := header["Forwarded"]
if ok {
header["Forwarded"] = []string{strings.Join(forwarded, ", ") + ", by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.host + "; proto=" + this.rawScheme}
header["Forwarded"] = []string{strings.Join(forwarded, ", ") + ", by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.ReqHost + "; proto=" + this.rawScheme}
} else {
header["Forwarded"] = []string{"by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.host + "; proto=" + this.rawScheme}
header["Forwarded"] = []string{"by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.ReqHost + "; proto=" + this.rawScheme}
}
}**/
@@ -1119,7 +1406,7 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
if this.reverseProxy != nil && this.reverseProxy.ShouldAddXForwardedHostHeader() {
if _, ok := header["X-Forwarded-Host"]; !ok {
this.RawReq.Header.Set("X-Forwarded-Host", this.Host)
this.RawReq.Header.Set("X-Forwarded-Host", this.ReqHost)
}
}
@@ -1159,7 +1446,7 @@ func (this *HTTPRequest) processRequestHeaders(reqHeader http.Header) {
}
// 域名
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.Host) {
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.ReqHost) {
continue
}
@@ -1243,7 +1530,7 @@ func (this *HTTPRequest) processResponseHeaders(statusCode int) {
}
// 域名
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.Host) {
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.ReqHost) {
continue
}
@@ -1277,13 +1564,13 @@ func (this *HTTPRequest) processResponseHeaders(statusCode int) {
// HSTS
if this.IsHTTPS &&
this.Server.HTTPS != nil &&
this.Server.HTTPS.SSLPolicy != nil &&
this.Server.HTTPS.SSLPolicy.IsOn &&
this.Server.HTTPS.SSLPolicy.HSTS != nil &&
this.Server.HTTPS.SSLPolicy.HSTS.IsOn &&
this.Server.HTTPS.SSLPolicy.HSTS.Match(this.Host) {
responseHeader.Set(this.Server.HTTPS.SSLPolicy.HSTS.HeaderKey(), this.Server.HTTPS.SSLPolicy.HSTS.HeaderValue())
this.ReqServer.HTTPS != nil &&
this.ReqServer.HTTPS.SSLPolicy != nil &&
this.ReqServer.HTTPS.SSLPolicy.IsOn &&
this.ReqServer.HTTPS.SSLPolicy.HSTS != nil &&
this.ReqServer.HTTPS.SSLPolicy.HSTS.IsOn &&
this.ReqServer.HTTPS.SSLPolicy.HSTS.Match(this.ReqHost) {
responseHeader.Set(this.ReqServer.HTTPS.SSLPolicy.HSTS.HeaderKey(), this.ReqServer.HTTPS.SSLPolicy.HSTS.HeaderValue())
}
}
@@ -1344,22 +1631,6 @@ func (this *HTTPRequest) canIgnore(err error) bool {
return false
}
// 关闭当前连接
func (this *HTTPRequest) closeConn() {
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
if requestConn == nil {
return
}
conn, ok := requestConn.(net.Conn)
if ok {
_ = conn.Close()
return
}
return
}
// 检查连接是否已关闭
func (this *HTTPRequest) isConnClosed() bool {
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)

View File

@@ -27,7 +27,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
subReq.ProtoMinor = this.RawReq.ProtoMinor
subReq.ProtoMajor = this.RawReq.ProtoMajor
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
subReq.Header.Set("Referer", this.requestFullURL())
subReq.Header.Set("Referer", this.URL())
var writer = NewEmptyResponseWriter(this.writer)
this.doSubRequest(writer, subReq)
return writer.StatusCode(), nil
@@ -45,7 +45,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.Host
headerValue += this.ReqHost
}
headerValue += "\""
if len(method.Charset) > 0 {

View File

@@ -5,10 +5,12 @@ import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
"net/http"
"path/filepath"
"strconv"
@@ -20,7 +22,7 @@ import (
func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
this.cacheCanTryStale = false
cachePolicy := this.Server.HTTPCachePolicy
var cachePolicy = this.ReqServer.HTTPCachePolicy
if cachePolicy == nil || !cachePolicy.IsOn {
return
}
@@ -105,12 +107,20 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
// TODO 支持Vary Header
// 缓存标签
var tags = []string{}
// 检查是否有缓存
key := this.Format(this.cacheRef.Key)
var key = this.Format(this.cacheRef.Key)
if len(key) == 0 {
this.cacheRef = nil
return
}
var method = this.Method()
if method != http.MethodGet && method != http.MethodPost {
key += cacheMethodSuffix + method
tags = append(tags, strings.ToLower(method))
}
this.cacheKey = key
this.varMapping["cache.key"] = key
@@ -121,22 +131,33 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
this.cacheRef = nil
return
}
this.writer.cacheStorage = storage
// 判断是否在Purge
if this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey {
this.varMapping["cache.status"] = "PURGE"
err := storage.Delete(key)
if err != nil {
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
var subKeys = []string{key, key + cacheMethodSuffix + "HEAD"}
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
subKeys = append(subKeys, key+compressionCacheSuffix+encoding)
subKeys = append(subKeys, key+webpCacheSuffix+compressionCacheSuffix+encoding)
}
for _, subKey := range subKeys {
err := storage.Delete(subKey)
if err != nil {
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
}
}
// 通过API节点清除别节点上的的Key
// TODO 改为队列不需要每个请求都使用goroutine
goman.New(func() {
rpcClient, err := rpc.SharedRPC()
if err == nil {
for _, rpcServerService := range rpcClient.ServerRPCList() {
_, err = rpcServerService.PurgeServerCache(rpcClient.Context(), &pb.PurgeServerCacheRequest{
Domains: []string{this.Host},
Domains: []string{this.ReqHost},
Keys: []string{key},
Prefixes: nil,
})
@@ -150,15 +171,57 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return true
}
// 调用回调
this.onRequest()
if this.writer.isFinished {
return
}
var reader caches.Reader
var err error
// 是否优先检查WebP
if this.web.WebP != nil &&
// 检查是否支持WebP
var webPIsEnabled = false
var isHeadMethod = method == http.MethodHead
if !isHeadMethod &&
this.web.WebP != nil &&
this.web.WebP.IsOn &&
this.web.WebP.MatchRequest(filepath.Ext(this.requestPath()), this.Format) &&
this.web.WebP.MatchAccept(this.requestHeader("Accept")) {
reader, _ = storage.OpenReader(key+webpSuffix, useStale)
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
this.web.WebP.MatchAccept(this.RawReq.Header.Get("Accept")) {
webPIsEnabled = true
}
// 检查压缩缓存
if !isHeadMethod && reader == nil {
if this.web.Compression != nil && this.web.Compression.IsOn {
_, encoding, ok := this.web.Compression.MatchAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"))
if ok {
// 检查支持WebP的压缩缓存
if webPIsEnabled {
reader, _ = storage.OpenReader(key+webpCacheSuffix+compressionCacheSuffix+encoding, useStale)
if reader != nil {
tags = append(tags, "webp", encoding)
}
}
// 检查普通缓存
if reader == nil {
reader, _ = storage.OpenReader(key+compressionCacheSuffix+encoding, useStale)
if reader != nil {
tags = append(tags, encoding)
}
}
}
}
}
// 检查WebP
if !isHeadMethod && reader == nil && webPIsEnabled {
reader, _ = storage.OpenReader(key+webpCacheSuffix, useStale)
if reader != nil {
this.writer.cacheReaderSuffix = webpCacheSuffix
tags = append(tags, "webp")
}
}
// 检查正常的文件
@@ -176,13 +239,16 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: open cache failed: "+err.Error())
}
return
}
}
defer func() {
_ = reader.Close()
if !this.writer.DelayRead() {
_ = reader.Close()
}
}()
if useStale {
@@ -223,7 +289,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
})
if err != nil {
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read header failed: "+err.Error())
}
return
}
@@ -238,6 +304,8 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
} else {
this.writer.Header().Set("X-Cache", "HIT, "+refType+", "+reader.TypeName())
}
} else {
this.writer.Header().Del("X-Cache")
}
if this.web.Cache.AddAgeHeader {
this.writer.Header().Set("Age", age)
@@ -249,7 +317,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var eTag = ""
var lastModifiedAt = reader.LastModified()
if lastModifiedAt > 0 {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
if len(tags) > 0 {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
} else {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
}
respHeader.Del("Etag")
respHeader["ETag"] = []string{eTag}
}
@@ -349,7 +421,6 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
}
respHeader := this.writer.Header()
if len(rangeSet) == 1 {
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(reader.BodySize(), 10))
respHeader.Set("Content-Length", strconv.FormatInt(rangeSet[0][1]-rangeSet[0][0]+1, 10))
@@ -371,7 +442,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return true
}
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
}
return
}
@@ -417,7 +488,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
})
if err != nil {
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
}
return true
}
@@ -431,21 +502,19 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return true
}
} else { // 没有Range
this.writer.PrepareCompression(reader.BodySize())
var resp = &http.Response{Body: reader}
this.writer.Prepare(resp, reader.BodySize(), reader.Status(), false)
this.writer.WriteHeader(reader.Status())
err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, errWritingToClient
}
return true, nil
})
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
if err == io.EOF {
err = nil
}
if err != nil {
this.varMapping["cache.status"] = "MISS"
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read body failed: "+err.Error())
}
return
}

View File

@@ -13,7 +13,7 @@ func (this *HTTPRequest) write404() {
this.processResponseHeaders(http.StatusNotFound)
this.writer.WriteHeader(http.StatusNotFound)
_, _ = this.writer.Write([]byte("404 page not found: '" + this.requestFullURL() + "'" + " (Request Id: " + this.requestId + ")"))
_, _ = this.writer.Write([]byte("404 page not found: '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
}
func (this *HTTPRequest) writeCode(code int) {
@@ -23,7 +23,7 @@ func (this *HTTPRequest) writeCode(code int) {
this.processResponseHeaders(code)
this.writer.WriteHeader(code)
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.requestFullURL() + "'" + " (Request Id: " + this.requestId + ")"))
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
}
func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {

View File

@@ -0,0 +1,11 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build !script
// +build !script
package nodes
func (this *HTTPRequest) onInit() {
}
func (this *HTTPRequest) onRequest() {
}

View File

@@ -52,13 +52,13 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
}
}
if !env.Has("SERVER_NAME") {
env["SERVER_NAME"] = this.Host
env["SERVER_NAME"] = this.ReqHost
}
if !env.Has("REQUEST_URI") {
env["REQUEST_URI"] = this.uri
}
if !env.Has("HOST") {
env["HOST"] = this.Host
env["HOST"] = this.ReqHost
}
if len(this.ServerAddr) > 0 {
@@ -149,7 +149,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
host, found := params["HTTP_HOST"]
if !found || len(host) == 0 {
params["HTTP_HOST"] = this.Host
params["HTTP_HOST"] = this.ReqHost
}
fcgiReq := fcgi.NewRequest()
@@ -190,7 +190,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
this.processResponseHeaders(resp.StatusCode)
// 准备
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)
// 设置响应代码
this.writer.WriteHeader(resp.StatusCode)

View File

@@ -13,7 +13,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
if this.web.MergeSlashes {
urlPath = utils.CleanPath(urlPath)
}
fullURL := this.requestScheme() + "://" + this.Host + urlPath
fullURL := this.requestScheme() + "://" + this.ReqHost + urlPath
for _, u := range this.web.HostRedirects {
if !u.IsOn {
continue
@@ -73,6 +73,13 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
return false
}
if u.KeepArgs {
var qIndex = strings.Index(this.uri, "?")
if qIndex >= 0 {
afterURL += this.uri[qIndex:]
}
}
if u.Status <= 0 {
this.processResponseHeaders(http.StatusTemporaryRedirect)
http.Redirect(this.RawWriter, this.RawReq, afterURL, http.StatusTemporaryRedirect)
@@ -88,12 +95,20 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
return false
}
var afterURL = u.AfterURL
if u.KeepArgs {
var qIndex = strings.Index(this.uri, "?")
if qIndex >= 0 {
afterURL += this.uri[qIndex:]
}
}
if u.Status <= 0 {
this.processResponseHeaders(http.StatusTemporaryRedirect)
http.Redirect(this.RawWriter, this.RawReq, u.AfterURL, http.StatusTemporaryRedirect)
http.Redirect(this.RawWriter, this.RawReq, afterURL, http.StatusTemporaryRedirect)
} else {
this.processResponseHeaders(u.Status)
http.Redirect(this.RawWriter, this.RawReq, u.AfterURL, u.Status)
http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
}
return true
}

View File

@@ -19,9 +19,9 @@ func (this *HTTPRequest) doRequestLimit() (shouldStop bool) {
if requestConn != nil {
clientConn, ok := requestConn.(ClientConnInterface)
if ok && !clientConn.IsBound() {
if !clientConn.Bind(this.Server.Id, this.requestRemoteAddr(true), this.web.RequestLimit.MaxConns, this.web.RequestLimit.MaxConnsPerIP) {
if !clientConn.Bind(this.ReqServer.Id, this.requestRemoteAddr(true), this.web.RequestLimit.MaxConns, this.web.RequestLimit.MaxConnsPerIP) {
this.writeCode(http.StatusTooManyRequests)
this.closeConn()
this.Close()
return true
}
}

View File

@@ -93,13 +93,13 @@ func (this *HTTPRequest) log() {
accessLog := &pb.HTTPAccessLog{
RequestId: this.requestId,
NodeId: sharedNodeConfig.Id,
ServerId: this.Server.Id,
ServerId: this.ReqServer.Id,
RemoteAddr: this.requestRemoteAddr(true),
RawRemoteAddr: addr,
RemotePort: int32(this.requestRemotePort()),
RemoteUser: this.requestRemoteUser(),
RequestURI: this.rawURI,
RequestPath: this.requestPath(),
RequestPath: this.Path(),
RequestLength: this.requestLength(),
RequestTime: this.requestCost,
RequestMethod: this.RawReq.Method,
@@ -114,7 +114,7 @@ func (this *HTTPRequest) log() {
TimeLocal: this.requestFromTime.Format("2/Jan/2006:15:04:05 -0700"),
Msec: float64(this.requestFromTime.Unix()) + float64(this.requestFromTime.Nanosecond())/1000000000,
Timestamp: this.requestFromTime.Unix(),
Host: this.Host,
Host: this.ReqHost,
Referer: referer,
UserAgent: userAgent,
Request: this.requestString(),

View File

@@ -50,7 +50,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
}
func (this *HTTPRequest) MetricServerId() int64 {
return this.Server.Id
return this.ReqServer.Id
}
func (this *HTTPRequest) MetricCategory() string {

View File

@@ -61,11 +61,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
if page.NewStatus > 0 {
// 自定义响应Headers
this.processResponseHeaders(page.NewStatus)
this.writer.Prepare(stat.Size(), page.NewStatus)
this.writer.Prepare(nil, stat.Size(), page.NewStatus, true)
this.writer.WriteHeader(page.NewStatus)
} else {
this.processResponseHeaders(status)
this.writer.Prepare(stat.Size(), status)
this.writer.Prepare(nil, stat.Size(), status, true)
this.writer.WriteHeader(status)
}
buf := utils.BytePool1k.Get()
@@ -100,11 +100,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
if page.NewStatus > 0 {
// 自定义响应Headers
this.processResponseHeaders(page.NewStatus)
this.writer.Prepare(int64(len(content)), page.NewStatus)
this.writer.Prepare(nil, int64(len(content)), page.NewStatus, true)
this.writer.WriteHeader(page.NewStatus)
} else {
this.processResponseHeaders(status)
this.writer.Prepare(int64(len(content)), status)
this.writer.Prepare(nil, int64(len(content)), status, true)
this.writer.WriteHeader(status)
}

View File

@@ -35,12 +35,12 @@ func (this *HTTPRequest) doReverseProxy() {
requestCall := shared.NewRequestCall()
requestCall.Request = this.RawReq
requestCall.Formatter = this.Format
requestCall.Domain = this.Host
requestCall.Domain = this.ReqHost
origin := this.reverseProxy.NextOrigin(requestCall)
requestCall.CallResponseCallbacks(this.writer)
if origin == nil {
err := errors.New(this.requestFullURL() + ": no available origin sites for reverse proxy")
remotelogs.ServerError(this.Server.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
err := errors.New(this.URL() + ": no available origin sites for reverse proxy")
remotelogs.ServerError(this.ReqServer.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
this.write50x(err, http.StatusBadGateway, true)
return
}
@@ -59,7 +59,7 @@ func (this *HTTPRequest) doReverseProxy() {
// 处理Scheme
if origin.Addr == nil {
err := errors.New(this.requestFullURL() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
err := errors.New(this.URL() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write50x(err, http.StatusBadGateway, true)
return
@@ -128,7 +128,7 @@ func (this *HTTPRequest) doReverseProxy() {
this.RawReq.Host = hostname
this.RawReq.URL.Host = this.RawReq.Host
} else {
this.RawReq.URL.Host = this.Host
this.RawReq.URL.Host = this.ReqHost
}
// 重组请求URL
@@ -146,6 +146,12 @@ func (this *HTTPRequest) doReverseProxy() {
this.setForwardHeaders(this.RawReq.Header)
this.processRequestHeaders(this.RawReq.Header)
// 调用回调
this.onRequest()
if this.writer.isFinished {
return
}
// 判断是否为Websocket请求
if this.RawReq.Header.Get("Upgrade") == "websocket" {
this.doWebsocket()
@@ -188,7 +194,9 @@ func (this *HTTPRequest) doReverseProxy() {
} else {
this.write50x(err, http.StatusBadGateway, true)
}
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
if httpErr.Err != io.EOF {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
}
} else {
// 是否为客户端方面的错误
isClientError := false
@@ -258,16 +266,23 @@ func (this *HTTPRequest) doReverseProxy() {
this.processResponseHeaders(resp.StatusCode)
// 是否需要刷新
shouldAutoFlush := this.reverseProxy.AutoFlush || this.RawReq.Header.Get("Accept") == "text/event-stream"
var shouldAutoFlush = this.reverseProxy.AutoFlush || this.RawReq.Header.Get("Accept") == "text/event-stream"
// 准备
delayHeaders := this.writer.Prepare(resp.ContentLength, resp.StatusCode)
var delayHeaders = this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)
// 设置响应代码
if !delayHeaders {
this.writer.WriteHeader(resp.StatusCode)
}
// 是否有内容
if resp.ContentLength == 0 && len(resp.TransferEncoding) == 0 {
_ = resp.Body.Close()
this.writer.SetOk()
return
}
// 输出到客户端
pool := this.bytePool(resp.ContentLength)
buf := pool.Get()

View File

@@ -15,7 +15,7 @@ func (this *HTTPRequest) doRewrite() (shouldShop bool) {
if this.rewriteRule.Mode == serverconfigs.HTTPRewriteModeProxy {
// 外部URL
if this.rewriteIsExternalURL {
host := this.Host
host := this.ReqHost
if len(this.rewriteRule.ProxyHost) > 0 {
host = this.rewriteRule.ProxyHost
}

View File

@@ -200,6 +200,12 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
respHeader.Set("ETag", eTag)
}
// 调用回调
this.onRequest()
if this.writer.isFinished {
return
}
// 支持 If-None-Match
if this.requestHeader("If-None-Match") == eTag {
// 自定义Header
@@ -296,7 +302,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.cacheRef = nil // 不支持缓存
}
this.writer.Prepare(fileSize, http.StatusOK)
this.writer.Prepare(nil, fileSize, http.StatusOK, true)
pool := this.bytePool(fileSize)
buf := pool.Get()

View File

@@ -6,11 +6,11 @@ import (
// 统计
func (this *HTTPRequest) doStat() {
if this.Server == nil {
if this.ReqServer == nil {
return
}
// 内置的统计
stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.Server.Id, this.requestRemoteAddr(true), this.writer.SentBodyBytes(), this.isAttack)
stats.SharedHTTPRequestStatManager.AddUserAgent(this.Server.Id, this.requestHeader("User-Agent"))
stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.ReqServer.Id, this.requestRemoteAddr(true), this.writer.SentBodyBytes(), this.isAttack)
stats.SharedHTTPRequestStatManager.AddUserAgent(this.ReqServer.Id, this.requestHeader("User-Agent"))
}

View File

@@ -10,8 +10,8 @@ func (this *HTTPRequest) doSubRequest(writer http.ResponseWriter, rawReq *http.R
req := &HTTPRequest{
RawReq: rawReq,
RawWriter: writer,
Server: this.Server,
Host: this.Host,
ReqServer: this.ReqServer,
ReqHost: this.ReqHost,
ServerName: this.ServerName,
ServerAddr: this.ServerAddr,
IsHTTP: this.IsHTTP,

View File

@@ -8,7 +8,7 @@ import (
// 流量限制
func (this *HTTPRequest) doTrafficLimit() {
var config = this.Server.TrafficLimit
var config = this.ReqServer.TrafficLimit
this.tags = append(this.tags, "bandwidth")

View File

@@ -54,9 +54,9 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
}
this.writer.AddHeaders(resp.Header)
if statusCode <= 0 {
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)
} else {
this.writer.Prepare(resp.ContentLength, statusCode)
this.writer.Prepare(resp, resp.ContentLength, statusCode, true)
}
// 设置响应代码

View File

@@ -153,3 +153,39 @@ func httpRequestNextId() string {
// timestamp + requestId + nodeId
return strconv.FormatInt(unixTime, 10) + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
}
// 检查是否可以接受某个编码
func httpAcceptEncoding(acceptEncodings string, encoding string) bool {
if len(acceptEncodings) == 0 {
return false
}
var pieces = strings.Split(acceptEncodings, ",")
for _, piece := range pieces {
var qualityIndex = strings.Index(piece, ";")
if qualityIndex >= 0 {
piece = piece[:qualityIndex]
}
if strings.TrimSpace(piece) == encoding {
return true
}
}
return false
}
// 分隔编码
func httpAcceptEncodings(acceptEncodings string) (encodings []string) {
if len(acceptEncodings) == 0 {
return
}
var pieces = strings.Split(acceptEncodings, ",")
for _, piece := range pieces {
var qualityIndex = strings.Index(piece, ";")
if qualityIndex >= 0 {
piece = piece[:qualityIndex]
}
encodings = append(encodings, strings.TrimSpace(piece))
}
return
}

View File

@@ -17,6 +17,10 @@ import (
// 调用WAF
func (this *HTTPRequest) doWAFRequest() (blocked bool) {
if this.web.FirewallRef == nil || !this.web.FirewallRef.IsOn {
return
}
var remoteAddr = this.requestRemoteAddr(true)
// 检查是否为白名单直连
@@ -31,16 +35,16 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
}
// 是否在全局名单中
if !iplibrary.AllowIP(remoteAddr, this.Server.Id) {
if !iplibrary.AllowIP(remoteAddr, this.ReqServer.Id) {
this.disableLog = true
this.closeConn()
this.Close()
return true
}
// 检查是否在临时黑名单中
if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeService, this.Server.Id, remoteAddr) || waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteAddr) {
if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeService, this.ReqServer.Id, remoteAddr) || waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteAddr) {
this.disableLog = true
this.closeConn()
this.Close()
return true
}
@@ -57,8 +61,8 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
}
// 公用的防火墙设置
if this.Server.HTTPFirewallPolicy != nil && this.Server.HTTPFirewallPolicy.IsOn {
blocked, breakChecking := this.checkWAFRequest(this.Server.HTTPFirewallPolicy)
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
blocked, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy)
if blocked {
return true
}
@@ -208,7 +212,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
}
// 添加统计
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Actions)
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
}
this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
@@ -219,6 +223,10 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// call response waf
func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
if this.web.FirewallRef == nil || !this.web.FirewallRef.IsOn {
return
}
// 当前服务的独立设置
if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp)
@@ -228,8 +236,8 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
}
// 公用的防火墙设置
if this.Server.HTTPFirewallPolicy != nil && this.Server.HTTPFirewallPolicy.IsOn {
blocked := this.checkWAFResponse(this.Server.HTTPFirewallPolicy, resp)
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp)
if blocked {
return true
}
@@ -266,7 +274,7 @@ func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFi
}
// 添加统计
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Actions)
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
}
this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
@@ -313,12 +321,12 @@ func (this *HTTPRequest) WAFRestoreBody(data []byte) {
// WAFServerId 服务ID
func (this *HTTPRequest) WAFServerId() int64 {
return this.Server.Id
return this.ReqServer.Id
}
// WAFClose 关闭连接
func (this *HTTPRequest) WAFClose() {
this.closeConn()
this.Close()
}
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {

File diff suppressed because it is too large Load Diff

View File

@@ -1,102 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"bufio"
"github.com/iwind/TeaGo/types"
"net"
"net/http"
"time"
)
// HTTPRateWriter 限速写入
type HTTPRateWriter struct {
parentWriter http.ResponseWriter
rateBytes int
lastBytes int
timeCost time.Duration
}
func NewHTTPRateWriter(writer http.ResponseWriter, rateBytes int64) http.ResponseWriter {
return &HTTPRateWriter{
parentWriter: writer,
rateBytes: types.Int(rateBytes),
}
}
func (this *HTTPRateWriter) Header() http.Header {
return this.parentWriter.Header()
}
func (this *HTTPRateWriter) Write(data []byte) (int, error) {
if len(data) == 0 {
return 0, nil
}
var left = this.rateBytes - this.lastBytes
if left <= 0 {
if this.timeCost > 0 && this.timeCost < 1*time.Second {
time.Sleep(1*time.Second - this.timeCost)
}
this.lastBytes = 0
this.timeCost = 0
return this.Write(data)
}
var n = len(data)
// n <= left
if n <= left {
this.lastBytes += n
var before = time.Now()
defer func() {
this.timeCost += time.Since(before)
}()
return this.parentWriter.Write(data)
}
// n > left
var before = time.Now()
result, err := this.parentWriter.Write(data[:left])
this.timeCost += time.Since(before)
if err != nil {
return result, err
}
this.lastBytes += left
return this.Write(data[left:])
}
func (this *HTTPRateWriter) WriteHeader(statusCode int) {
this.parentWriter.WriteHeader(statusCode)
}
// Hijack Hijack
func (this *HTTPRateWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
if this.parentWriter == nil {
return
}
hijack, ok := this.parentWriter.(http.Hijacker)
if ok {
return hijack.Hijack()
}
return
}
// Flush Flush
func (this *HTTPRateWriter) Flush() {
if this.parentWriter == nil {
return
}
flusher, ok := this.parentWriter.(http.Flusher)
if ok {
flusher.Flush()
return
}
}

View File

@@ -60,8 +60,8 @@ func (this *Listener) listenTCP() error {
if err != nil {
return err
}
var netListener = NewClientListener1(tcpListener, protocol.IsHTTPFamily() || protocol.IsHTTPSFamily())
events.On(events.EventQuit, func() {
var netListener = NewClientListener(tcpListener, protocol.IsHTTPFamily() || protocol.IsHTTPSFamily())
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("LISTENER", "quit "+this.group.FullAddr())
_ = netListener.Close()
})
@@ -122,7 +122,7 @@ func (this *Listener) listenUDP() error {
if err != nil {
return err
}
events.On(events.EventQuit, func() {
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("LISTENER", "quit "+this.group.FullAddr())
_ = listener.Close()
})
@@ -143,6 +143,8 @@ func (this *Listener) listenUDP() error {
}
func (this *Listener) Close() error {
events.Remove(this)
if this.listener == nil {
return nil
}

View File

@@ -25,8 +25,8 @@ func (this *BaseListener) Reset() {
}
// CountActiveListeners 获取当前活跃连接数
func (this *BaseListener) CountActiveListeners() int {
// CountActiveConnections 获取当前活跃连接数
func (this *BaseListener) CountActiveConnections() int {
return types.Int(this.countActiveConnections)
}

View File

@@ -208,8 +208,8 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
req := &HTTPRequest{
RawReq: rawReq,
RawWriter: rawWriter,
Server: server,
Host: reqHost,
ReqServer: server,
ReqHost: reqHost,
ServerName: serverName,
ServerAddr: this.addr,
IsHTTP: this.isHTTP,

View File

@@ -16,6 +16,6 @@ type ListenerInterface interface {
// Reload 重载配置
Reload(serverGroup *serverconfigs.ServerAddressGroup)
// CountActiveListeners 获取当前活跃的连接数
CountActiveListeners() int
// CountActiveConnections 获取当前活跃的连接数
CountActiveConnections() int
}

View File

@@ -14,6 +14,7 @@ import (
"os/exec"
"regexp"
"runtime"
"sort"
"strings"
"sync"
"time"
@@ -29,6 +30,8 @@ type ListenerManager struct {
retryListenerMap map[string]*Listener // 需要重试的监听器 addr => Listener
ticker *time.Ticker
lastPortStrings string
}
// NewListenerManager 获取新对象
@@ -143,6 +146,9 @@ func (this *ListenerManager) Start(node *nodeconfigs.NodeConfig) error {
}
}
// 加入到firewalld
this.addToFirewalld(groupAddrs)
return nil
}
@@ -153,7 +159,7 @@ func (this *ListenerManager) TotalActiveConnections() int {
total := 0
for _, listener := range this.listenersMap {
total += listener.listener.CountActiveListeners()
total += listener.listener.CountActiveConnections()
}
return total
}
@@ -214,3 +220,63 @@ func (this *ListenerManager) findProcessNameWithPort(isUdp bool, port string) st
}
return ""
}
func (this *ListenerManager) addToFirewalld(groupAddrs []string) {
if !sharedNodeConfig.AutoOpenPorts {
return
}
// 组合端口号
var ports = []string{}
for _, addr := range groupAddrs {
var protocol = "tcp"
if strings.HasPrefix(addr, "udp") {
protocol = "udp"
}
var lastIndex = strings.LastIndex(addr, ":")
if lastIndex > 0 {
var portString = addr[lastIndex+1:]
ports = append(ports, portString+"/"+protocol)
}
}
if len(ports) == 0 {
return
}
// 检查是否有变化
sort.Strings(ports)
var newPortStrings = strings.Join(ports, ",")
if newPortStrings == this.lastPortStrings {
return
}
this.lastPortStrings = newPortStrings
firewallCmd, err := exec.LookPath("firewall-cmd")
if err != nil || len(firewallCmd) == 0 {
return
}
remotelogs.Println("FIREWALLD", "open ports automatically")
for _, port := range ports {
{
// TODO 需要支持sudo
var cmd = exec.Command(firewallCmd, "--add-port="+port, "--permanent")
err = cmd.Run()
if err != nil {
remotelogs.Warn("FIREWALLD", "'"+cmd.String()+"': "+err.Error())
return
}
}
{
// TODO 需要支持sudo
var cmd = exec.Command(firewallCmd, "--add-port="+port)
err = cmd.Run()
if err != nil {
remotelogs.Warn("FIREWALLD", "'"+cmd.String()+"': "+err.Error())
return
}
}
}
}

View File

@@ -11,6 +11,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/configs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
@@ -55,12 +56,16 @@ type Node struct {
maxCPU int32
maxThreads int
timezone string
updatingServerMap map[int64]*serverconfigs.ServerConfig
}
func NewNode() *Node {
return &Node{
sock: gosock.NewTmpSock(teaconst.ProcessName),
maxThreads: -1,
sock: gosock.NewTmpSock(teaconst.ProcessName),
maxThreads: -1,
maxCPU: -1,
updatingServerMap: map[int64]*serverconfigs.ServerConfig{},
}
}
@@ -262,7 +267,7 @@ func (this *Node) loop() error {
defer tr.End()
// 检查api.yaml是否存在
apiConfigFile := Tea.ConfigFile("api.yaml")
var apiConfigFile = Tea.ConfigFile("api.yaml")
_, err := os.Stat(apiConfigFile)
if err != nil {
return nil
@@ -273,7 +278,7 @@ func (this *Node) loop() error {
return errors.New("create rpc client failed: " + err.Error())
}
nodeCtx := rpcClient.Context()
var nodeCtx = rpcClient.Context()
tasksResp, err := rpcClient.NodeTaskRPC().FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
if err != nil {
return errors.New("read node tasks failed: " + err.Error())
@@ -293,11 +298,15 @@ func (this *Node) loop() error {
return err
}
case "configChanged":
if !task.IsPrimary {
// 我们等等主节点配置准备完毕
time.Sleep(2 * time.Second)
if task.ServerId > 0 {
err = this.syncServerConfig(task.ServerId)
} else {
if !task.IsPrimary {
// 我们等等主节点配置准备完毕
time.Sleep(2 * time.Second)
}
err = this.syncConfig(task.Version)
}
err := this.syncConfig(task.Version)
if err != nil {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
@@ -314,6 +323,7 @@ func (this *Node) loop() error {
if err != nil {
return err
}
case "nodeVersionChanged":
goman.New(func() {
sharedUpgradeManager.Start()
@@ -417,22 +427,8 @@ func (this *Node) syncConfig(taskVersion int64) error {
remotelogs.Println("NODE", "loading config ...")
}
nodeconfigs.ResetNodeConfig(nodeConfig)
caches.SharedManager.MaxDiskCapacity = nodeConfig.MaxCacheDiskCapacity
caches.SharedManager.MaxMemoryCapacity = nodeConfig.MaxCacheMemoryCapacity
if len(nodeConfig.HTTPCachePolicies) > 0 {
caches.SharedManager.UpdatePolicies(nodeConfig.HTTPCachePolicies)
} else {
caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
}
sharedWAFManager.UpdatePolicies(nodeConfig.FindAllFirewallPolicies())
iplibrary.SharedActionManager.UpdateActions(nodeConfig.FirewallActions)
sharedNodeConfig = nodeConfig
this.onReload(nodeConfig)
metrics.SharedManager.Update(nodeConfig.MetricItems)
// 发送事件
events.Notify(events.EventReload)
@@ -445,30 +441,96 @@ func (this *Node) syncConfig(taskVersion int64) error {
return nil
}
// 读取单个服务配置
func (this *Node) syncServerConfig(serverId int64) error {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.ServerRPC().ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
if err != nil {
return err
}
this.locker.Lock()
defer this.locker.Unlock()
if len(resp.ServerConfigJSON) == 0 {
this.updatingServerMap[serverId] = nil
} else {
var config = &serverconfigs.ServerConfig{}
err = json.Unmarshal(resp.ServerConfigJSON, config)
if err != nil {
return err
}
this.updatingServerMap[serverId] = config
}
return nil
}
// 启动同步计时器
func (this *Node) startSyncTimer() {
// TODO 这个时间间隔可以自行设置
ticker := time.NewTicker(60 * time.Second)
events.On(events.EventQuit, func() {
var taskTicker = time.NewTicker(60 * time.Second)
var serverChangeTicker = time.NewTicker(5 * time.Second)
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("NODE", "quit sync timer")
ticker.Stop()
taskTicker.Stop()
serverChangeTicker.Stop()
})
goman.New(func() {
for {
select {
case <-ticker.C:
case <-taskTicker.C: // 定期执行
err := this.loop()
if err != nil {
remotelogs.Error("NODE", "sync config error: "+err.Error())
continue
}
case <-nodeTaskNotify:
case <-serverChangeTicker.C: // 服务变化
this.locker.Lock()
if len(this.updatingServerMap) > 0 {
var updatingServerMap = this.updatingServerMap
this.updatingServerMap = map[int64]*serverconfigs.ServerConfig{}
newNodeConfig, err := nodeconfigs.CloneNodeConfig(sharedNodeConfig)
if err != nil {
remotelogs.Error("NODE", "apply server config error: "+err.Error())
continue
}
for serverId, serverConfig := range updatingServerMap {
if serverConfig != nil {
newNodeConfig.AddServer(serverConfig)
} else {
newNodeConfig.RemoveServer(serverId)
}
}
err, serverErrors := newNodeConfig.Init()
if err != nil {
remotelogs.Error("NODE", "apply server config error: "+err.Error())
continue
}
if len(serverErrors) > 0 {
for _, serverErr := range serverErrors {
remotelogs.ServerError(serverErr.Id, "NODE", serverErr.Message, nodeconfigs.NodeLogTypeServerConfigInitFailed, maps.Map{})
}
}
this.onReload(newNodeConfig)
err = sharedListenerManager.Start(newNodeConfig)
if err != nil {
remotelogs.Error("NODE", "apply server config error: "+err.Error())
}
}
this.locker.Unlock()
case <-nodeTaskNotify: // 有新的更新任务
err := this.loop()
if err != nil {
remotelogs.Error("NODE", "sync config error: "+err.Error())
continue
}
case <-nodeConfigChangedNotify:
case <-nodeConfigChangedNotify: // 节点变化通知
err := this.syncConfig(0)
if err != nil {
remotelogs.Error("NODE", "sync config error: "+err.Error())
@@ -635,6 +697,51 @@ func (this *Node) listenSock() error {
"limiter": sharedConnectionsLimiter.Len(),
},
})
case "dropIP":
var m = maps.NewMap(cmd.Params)
var ip = m.GetString("ip")
var timeSeconds = m.GetInt("timeoutSeconds")
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds)
if err != nil {
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"error": err.Error(),
},
})
} else {
_ = cmd.ReplyOk()
}
case "rejectIP":
var m = maps.NewMap(cmd.Params)
var ip = m.GetString("ip")
var timeSeconds = m.GetInt("timeoutSeconds")
err := firewalls.Firewall().RejectSourceIP(ip, timeSeconds)
if err != nil {
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"error": err.Error(),
},
})
} else {
_ = cmd.ReplyOk()
}
case "removeIP":
var m = maps.NewMap(cmd.Params)
var ip = m.GetString("ip")
err := firewalls.Firewall().RemoveSourceIP(ip)
if err != nil {
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"error": err.Error(),
},
})
} else {
_ = cmd.ReplyOk()
}
case "gc":
runtime.GC()
debug.FreeOSMemory()
_ = cmd.ReplyOk()
}
})
@@ -644,7 +751,7 @@ func (this *Node) listenSock() error {
}
})
events.On(events.EventQuit, func() {
events.OnKey(events.EventQuit, this, func() {
logs.Println("NODE", "quit unix sock")
_ = this.sock.Close()
})
@@ -654,14 +761,34 @@ func (this *Node) listenSock() error {
// 重载配置调用
func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
nodeconfigs.ResetNodeConfig(config)
sharedNodeConfig = config
// 缓存策略
caches.SharedManager.MaxDiskCapacity = config.MaxCacheDiskCapacity
caches.SharedManager.MaxMemoryCapacity = config.MaxCacheMemoryCapacity
if len(config.HTTPCachePolicies) > 0 {
caches.SharedManager.UpdatePolicies(config.HTTPCachePolicies)
} else {
caches.SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
}
// WAF策略
sharedWAFManager.UpdatePolicies(config.FindAllFirewallPolicies())
iplibrary.SharedActionManager.UpdateActions(config.FirewallActions)
// 统计指标
metrics.SharedManager.Update(config.MetricItems)
// max cpu
if config.MaxCPU != this.maxCPU {
if config.MaxCPU > 0 && config.MaxCPU < int32(runtime.NumCPU()) {
runtime.GOMAXPROCS(int(config.MaxCPU))
remotelogs.Println("NODE", "[CPU]set max cpu to '"+types.String(config.MaxCPU)+"'")
} else {
runtime.GOMAXPROCS(runtime.NumCPU())
remotelogs.Println("NODE", "[CPU]set max cpu to '"+types.String(runtime.NumCPU())+"'")
var threads = runtime.NumCPU() * 4
runtime.GOMAXPROCS(threads)
remotelogs.Println("NODE", "[CPU]set max cpu to '"+types.String(threads)+"'")
}
this.maxCPU = config.MaxCPU
@@ -707,4 +834,9 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
time.Local = location
this.timezone = timeZone
}
// product information
if config.ProductConfig != nil {
teaconst.GlobalProductName = config.ProductConfig.Name
}
}

View File

@@ -41,9 +41,9 @@ func (this *NodeStatusExecutor) Listen() {
this.update()
// TODO 这个时间间隔可以配置
ticker := time.NewTicker(30 * time.Second)
var ticker = time.NewTicker(30 * time.Second)
events.On(events.EventQuit, func() {
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("NODE_STATUS", "quit executor")
ticker.Stop()
})

Some files were not shown because too many files have changed in this diff Show More