Compare commits

...

74 Commits

Author SHA1 Message Date
刘祥超
01cfccebbd 缓存结束后增加Content-Length对比 2021-07-20 19:48:08 +08:00
刘祥超
6bd7da5e6e build脚本增加arm编译 2021-07-20 19:01:49 +08:00
刘祥超
dcba9c2f3e 调整格式等 2021-07-20 19:01:37 +08:00
刘祥超
f38e80e82d 自动替换API节点时增加对新节点的测试 2021-07-20 18:17:25 +08:00
刘祥超
9bd38094c3 将WAF模板中的cc修改为cc2 2021-07-19 11:01:38 +08:00
刘祥超
7e37fc3b80 实现新的CC 2021-07-19 10:49:56 +08:00
刘祥超
d775dfeeaa WAF增加多个动作 2021-07-18 15:51:49 +08:00
刘祥超
0486f86898 增加攻击拦截统计 2021-07-13 11:04:38 +08:00
刘祥超
102157c893 节点看板增加缓存目录用量 2021-07-08 19:43:30 +08:00
刘祥超
dbd92368ae 文件缓存统计去除过期时间条件 2021-07-08 09:19:41 +08:00
刘祥超
9e418e73bf 节点状态上报流量 2021-07-07 15:18:01 +08:00
刘祥超
5d40eec163 实现节点看板(仅对企业版开放) 2021-07-06 20:06:57 +08:00
刘祥超
1a7a67238d 增加域名统计 2021-07-05 11:36:50 +08:00
刘祥超
6707437bae 指标数据增加总和数据 2021-07-01 10:39:56 +08:00
刘祥超
df7859387d 实现基础的统计指标 2021-06-30 20:01:00 +08:00
刘祥超
889c52330d 修改版本为0.2.5 2021-06-28 10:32:18 +08:00
刘祥超
0e912b79cd TOA通讯失败时,关闭连接 2021-06-27 17:31:10 +08:00
刘祥超
12f3916e45 ip2region增加IP格式检查 2021-06-27 17:30:45 +08:00
刘祥超
7813e2c3d2 更新TOA 2021-06-24 17:38:29 +08:00
刘祥超
54eff9bfae 删除TOA 2021-06-24 17:29:40 +08:00
刘祥超
635cdd4338 上传TOA编译文件 2021-06-24 16:59:52 +08:00
刘祥超
4c64d3ab0f 实现公用的IP名单 2021-06-23 13:14:37 +08:00
刘祥超
93a5c90fcb 应用网站自定义的WAF出站规则 2021-06-21 15:29:07 +08:00
刘祥超
eb5e863146 变更版本 2021-06-21 14:43:29 +08:00
刘祥超
78e566174f 更新部分Go Package 2021-06-20 19:23:35 +08:00
刘祥超
dd93a93ba9 访问控制支持基本认证和子请求认证 2021-06-19 21:35:57 +08:00
刘祥超
aaa6899976 优化文件缓存 2021-06-17 21:13:21 +08:00
刘祥超
e04e3287b4 调整版本为0.2.3 2021-06-17 18:45:59 +08:00
刘祥超
e715693156 调整版本为0.2.2 2021-06-17 18:05:35 +08:00
刘祥超
2798c3c5e5 修复移除内存缓存死锁的Bug 2021-06-17 18:04:56 +08:00
刘祥超
489e081720 修复可能导致死锁冲突的Bug 2021-06-16 16:10:02 +08:00
刘祥超
77a8eb5c1a 触发浏览器304缓存也算缓存命中 2021-06-16 11:04:19 +08:00
刘祥超
3b7d2b91c7 调整版本为0.2.1 2021-06-16 08:30:26 +08:00
刘祥超
20b299fb3b 优化错误提示 2021-06-16 08:29:38 +08:00
刘祥超
e4b3d2b2aa 不提示一些客户端错误 2021-06-16 07:52:06 +08:00
刘祥超
d237ee6b5b 节点启动错误时自动尝试从本地读取缓存数据 2021-06-15 10:55:49 +08:00
刘祥超
34aa6125df 修改一处注释错别字 2021-06-15 10:47:40 +08:00
刘祥超
24fc2249bb 优化文件缓存 2021-06-14 19:55:06 +08:00
刘祥超
84c931b411 缓存支持ETag和Last-Modified 2021-06-14 11:46:39 +08:00
刘祥超
7f422a2946 优化cache/FileList错误提示 2021-06-13 17:51:04 +08:00
刘祥超
13194366a5 优化文件缓存 2021-06-13 17:37:57 +08:00
刘祥超
993cda7766 修复内存缓存没有init()的Bug 2021-06-12 10:03:33 +08:00
刘祥超
a46e970c74 优化内存缓存 2021-06-11 14:53:51 +08:00
刘祥超
085adcf1c4 实现节点自动升级成最新版本 2021-06-10 19:19:15 +08:00
刘祥超
c1af8b36a4 完成两个TODO文档说明 2021-06-10 11:35:20 +08:00
刘祥超
8cba12b4b5 URL跳转规则支持匹配条件 2021-06-09 21:44:59 +08:00
刘祥超
3debe1d1df 支持不缓存条件 2021-06-08 22:45:11 +08:00
刘祥超
549f110e5f 增加服务流量统计 2021-06-08 11:24:41 +08:00
刘祥超
f3a45e9e64 改进UDP IsOk的使用方法 2021-06-07 15:48:39 +08:00
刘祥超
f461760158 支持UDP代理 2021-06-07 15:45:47 +08:00
刘祥超
a49b724745 优化HTTP缓存,主要是并发冲突、缓存写入不全等问题 2021-06-06 23:42:11 +08:00
刘祥超
0df5dfad23 某个服务端口启动失败后,会自动重试 2021-06-06 13:40:00 +08:00
刘祥超
aeb1bc08a7 改进编辑脚本 2021-06-06 11:58:41 +08:00
刘祥超
c51aca621a 调整API命名 2021-06-01 19:52:37 +08:00
刘祥超
c78d055dae 更新fcgi 2021-05-28 14:00:45 +08:00
刘祥超
4502a3b132 改进清空缓存目录逻辑 2021-05-25 18:28:24 +08:00
刘祥超
fa99d86d6f 对部分错误提示降级 2021-05-25 11:16:05 +08:00
刘祥超
2edd2bb105 缓存文件列表初始化时自动创建目录 2021-05-25 11:06:43 +08:00
刘祥超
0f8aee0ccb 修改版本号 2021-05-25 11:06:07 +08:00
刘祥超
2500929a99 调整在Mac OS上的编译脚本 2021-05-25 11:05:55 +08:00
刘祥超
496ee6cfa0 优化代码 2021-05-24 09:37:37 +08:00
刘祥超
437914a321 支持缓存策略全局的缓存条件/X-Cache中加入更多信息 2021-05-24 09:23:51 +08:00
刘祥超
3a93bf756a 加快缓存策略启动速度 2021-05-23 22:59:00 +08:00
刘祥超
38d81f340e 调整个别日志级别 2021-05-23 20:45:14 +08:00
刘祥超
889b9d063a URL跳转支持正则匹配 2021-05-23 17:01:08 +08:00
刘祥超
4c73b3618f 优化代码 2021-05-23 16:16:56 +08:00
刘祥超
df5f50682a 不再提示http2 Stream相关错误 2021-05-23 15:50:21 +08:00
刘祥超
9545bf69db 删除一个注释 2021-05-23 14:31:25 +08:00
刘祥超
b2f18c22ee 修复缓存状态码不生效的问题 2021-05-23 14:29:56 +08:00
刘祥超
63e3b7ac2f 修复跳转到HTTPS的自定义端口无法起作用的Bug 2021-05-22 10:26:37 +08:00
刘祥超
760a62c286 修改两处日志级别 2021-05-22 10:26:12 +08:00
刘祥超
296848a6d6 优化一个文件缓存统计的Bug 2021-05-19 22:14:57 +08:00
刘祥超
4e04534244 更改版本号 2021-05-19 14:16:04 +08:00
刘祥超
cad43e610d 缓存文件列表使用sqlite管理 2021-05-19 12:07:35 +08:00
173 changed files with 6441 additions and 1604 deletions

View File

@@ -33,6 +33,7 @@ function build() {
mkdir $DIST/bin
mkdir $DIST/configs
mkdir $DIST/logs
mkdir $DIST/data
fi
cp $ROOT/configs/api.template.yaml $DIST/configs
@@ -47,7 +48,42 @@ function build() {
fi
echo "building ..."
env GOOS=${OS} GOARCH=${ARCH} go build -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
MUSL_DIR="/usr/local/opt/musl-cross/bin"
CC_PATH=""
CXX_PATH=""
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
# /usr/local/opt/musl-cross/bin/
if [ "${ARCH}" == "amd64" ]; then
CC_PATH="x86_64-linux-musl-gcc"
CXX_PATH="x86_64-linux-musl-g++"
fi
if [ "${ARCH}" == "386" ]; then
CC_PATH="i486-linux-musl-gcc"
CXX_PATH="i486-linux-musl-g++"
fi
if [ "${ARCH}" == "arm64" ]; then
CC_PATH="aarch64-linux-musl-gcc"
CXX_PATH="aarch64-linux-musl-g++"
fi
if [ "${ARCH}" == "arm" ]; then
CC_PATH="arm-linux-musleabi-gcc"
CXX_PATH="arm-linux-musleabi-g++"
fi
if [ "${ARCH}" == "mips64" ]; then
CC_PATH="mips64-linux-musl-gcc"
CXX_PATH="mips64-linux-musl-g++"
fi
if [ "${ARCH}" == "mips64le" ]; then
CC_PATH="mips64el-linux-musl-gcc"
CXX_PATH="mips64el-linux-musl-g++"
fi
fi
if [ ! -z $CC_PATH ]; then
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
else
env GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
fi
# delete hidden files
find $DIST -name ".DS_Store" -delete

1
build/data/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*

View File

@@ -1 +0,0 @@
edge-toa

BIN
build/edge-toa/edge-toa Executable file

Binary file not shown.

View File

@@ -7,8 +7,11 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/nodes"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"net/http"
_ "net/http/pprof"
"os"
"syscall"
)
@@ -17,7 +20,7 @@ func main() {
app := apps.NewAppCmd().
Version(teaconst.Version).
Product(teaconst.ProductName).
Usage(teaconst.ProcessName + " [-v|start|stop|restart|quit|test|service|daemon]")
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|service|daemon|pprof]")
app.On("test", func() {
err := nodes.NewNode().Test()
@@ -57,6 +60,21 @@ func main() {
_ = process.Signal(syscall.SIGQUIT)
}
})
app.On("pprof", func() {
// TODO 自己指定端口
addr := "127.0.0.1:6060"
logs.Println("starting with pprof '" + addr + "'...")
go func() {
err := http.ListenAndServe(addr, nil)
if err != nil {
logs.Println("[error]" + err.Error())
}
}()
node := nodes.NewNode()
node.Start()
})
app.Run(func() {
node := nodes.NewNode()
node.Start()

18
go.mod
View File

@@ -11,14 +11,18 @@ require (
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/golang/protobuf v1.4.2
github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f
github.com/iwind/gofcgi v0.0.0-20210506081859-17498ab3e9d7
github.com/golang/protobuf v1.5.2
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/mssola/user_agent v0.5.2
github.com/shirou/gopsutil v2.20.9+incompatible
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
google.golang.org/grpc v1.32.0
github.com/shirou/gopsutil v3.21.5+incompatible
github.com/tklauser/go-sysconf v0.3.6 // indirect
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
golang.org/x/text v0.3.6
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
google.golang.org/grpc v1.38.0
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
)

75
go.sum
View File

@@ -11,9 +11,11 @@ github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiU
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -23,6 +25,7 @@ github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cu
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -45,28 +48,34 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f h1:6Ws2H+eorfVUoMO2jta6A9nIdh8oi5/5LXo/LkAxR+E=
github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/gofcgi v0.0.0-20210506081859-17498ab3e9d7 h1:apv23QzWNmv0D76gB3+u/5kf0F/Yw4W8h489CWUZtss=
github.com/iwind/gofcgi v0.0.0-20210506081859-17498ab3e9d7/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060 h1:qdLtK4PDXxk2vMKkTWl5Fl9xqYuRCukzWAgJbLHdfOo=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -87,14 +96,20 @@ github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mo
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/shirou/gopsutil v2.20.9+incompatible h1:msXs2frUV+O/JLva9EDLpuJ84PrFsdCTCQex8PUdtkQ=
github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -106,9 +121,11 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -117,12 +134,15 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -134,34 +154,51 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -170,8 +207,10 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
@@ -179,10 +218,12 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

41
internal/caches/errors.go Normal file
View File

@@ -0,0 +1,41 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import "errors"
// 常用的几个错误
var (
ErrNotFound = errors.New("cache not found")
ErrFileIsWriting = errors.New("the file is writing")
ErrInvalidRange = errors.New("invalid range")
)
// CapacityError 容量错误
// 独立出来是为了可以在有些场合下可以忽略,防止产生没必要的错误提示数量太多
type CapacityError struct {
err string
}
func NewCapacityError(err string) error {
return &CapacityError{err: err}
}
func (this *CapacityError) Error() string {
return this.err
}
// CanIgnoreErr 检查错误是否可以忽略
func CanIgnoreErr(err error) bool {
if err == nil {
return true
}
if err == ErrFileIsWriting {
return true
}
_, ok := err.(*CapacityError)
if ok {
return true
}
return false
}

View File

@@ -0,0 +1,16 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"github.com/iwind/TeaGo/assert"
"testing"
)
func TestCanIgnoreErr(t *testing.T) {
a := assert.NewAssertion(t)
a.IsTrue(CanIgnoreErr(ErrFileIsWriting))
a.IsTrue(CanIgnoreErr(NewCapacityError("over capcity")))
a.IsFalse(CanIgnoreErr(ErrNotFound))
}

View File

@@ -1,6 +1,8 @@
package caches
import "time"
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
)
type ItemType = int
@@ -10,20 +12,22 @@ const (
)
type Item struct {
Type ItemType
Key string
ExpiredAt int64
HeaderSize int64
BodySize int64
MetaSize int64
Type ItemType `json:"type"`
Key string `json:"key"`
ExpiredAt int64 `json:"expiredAt"`
HeaderSize int64 `json:"headerSize"`
BodySize int64 `json:"bodySize"`
MetaSize int64 `json:"metaSize"`
Host string `json:"host"` // 主机名
ServerId int64 `json:"serverId"` // 服务ID
}
func (this *Item) IsExpired() bool {
return this.ExpiredAt < time.Now().Unix()
return this.ExpiredAt < utils.UnixTime()
}
func (this *Item) TotalSize() int64 {
return this.Size() + this.MetaSize + int64(len(this.Key))
return this.Size() + this.MetaSize + int64(len(this.Key)) + 64
}
func (this *Item) Size() int64 {

View File

@@ -1,145 +0,0 @@
package caches
import (
"strings"
"sync"
)
// 缓存列表管理
type List struct {
m map[string]*Item // hash => item
locker sync.RWMutex
onAdd func(item *Item)
onRemove func(item *Item)
}
func NewList() *List {
return &List{
m: map[string]*Item{},
}
}
func (this *List) Reset() {
this.locker.Lock()
this.m = map[string]*Item{}
this.locker.Unlock()
}
func (this *List) Add(hash string, item *Item) {
this.locker.Lock()
if this.onAdd != nil {
this.onAdd(item)
}
this.m[hash] = item
this.locker.Unlock()
}
func (this *List) Exist(hash string) bool {
this.locker.RLock()
defer this.locker.RUnlock()
item, ok := this.m[hash]
if !ok {
return false
}
return !item.IsExpired()
}
// 根据前缀进行查找
func (this *List) FindKeysWithPrefix(prefix string) (keys []string) {
this.locker.RLock()
defer this.locker.RUnlock()
// TODO 需要优化性能支持千万级数据低于1s的处理速度
for _, item := range this.m {
if strings.HasPrefix(item.Key, prefix) {
keys = append(keys, item.Key)
}
}
return
}
func (this *List) Remove(hash string) {
this.locker.Lock()
item, ok := this.m[hash]
if ok {
if this.onRemove != nil {
this.onRemove(item)
}
delete(this.m, hash)
}
this.locker.Unlock()
}
// 清理过期的缓存
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用
func (this *List) Purge(count int, callback func(hash string)) {
this.locker.Lock()
deletedHashList := []string{}
for hash, item := range this.m {
if count <= 0 {
break
}
if item.IsExpired() {
if this.onRemove != nil {
this.onRemove(item)
}
delete(this.m, hash)
deletedHashList = append(deletedHashList, hash)
}
count--
}
this.locker.Unlock()
// 执行外部操作
for _, hash := range deletedHashList {
if callback != nil {
callback(hash)
}
}
}
func (this *List) Stat(check func(hash string) bool) *Stat {
this.locker.RLock()
defer this.locker.RUnlock()
result := &Stat{
Count: 0,
Size: 0,
}
for hash, item := range this.m {
if !item.IsExpired() {
// 检查文件是否存在、内容是否正确等
if check != nil && check(hash) {
result.Count++
result.ValueSize += item.Size()
result.Size += item.TotalSize()
}
}
}
return result
}
// 总数量
func (this *List) Count() int64 {
this.locker.RLock()
count := int64(len(this.m))
this.locker.RUnlock()
return count
}
// 添加事件
func (this *List) OnAdd(f func(item *Item)) {
this.onAdd = f
}
// 删除事件
func (this *List) OnRemove(f func(item *Item)) {
this.onRemove = f
}

View File

@@ -0,0 +1,423 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"database/sql"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/lists"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
"sync/atomic"
"time"
)
// FileList 文件缓存列表管理
type FileList struct {
dir string
db *sql.DB
total int64
onAdd func(item *Item)
onRemove func(item *Item)
existsByHashStmt *sql.Stmt // 根据hash检查是否存在
insertStmt *sql.Stmt // 写入数据
selectByHashStmt *sql.Stmt // 使用hash查询数据
deleteByHashStmt *sql.Stmt // 根据hash删除数据
statStmt *sql.Stmt // 统计
purgeStmt *sql.Stmt // 清理
deleteAllStmt *sql.Stmt // 删除所有数据
oldTables []string
itemsTableName string
isClosed bool
}
func NewFileList(dir string) ListInterface {
return &FileList{dir: dir}
}
func (this *FileList) Init() error {
// 检查目录是否存在
_, err := os.Stat(this.dir)
if err != nil {
err = os.MkdirAll(this.dir, 0777)
if err != nil {
return err
}
remotelogs.Println("CACHE", "create cache dir '"+this.dir+"'")
}
this.itemsTableName = "cacheItems_v2"
db, err := sql.Open("sqlite3", "file:"+this.dir+"/index.db?cache=shared&mode=rwc&_journal_mode=WAL")
if err != nil {
return err
}
db.SetMaxOpenConns(1)
this.db = db
// 清除旧表
this.oldTables = []string{
"cacheItems",
}
err = this.removeOldTables()
if err != nil {
remotelogs.Warn("CACHE", "clean old tables failed: "+err.Error())
}
// TODO 耗时过长,暂时不整理数据库
/**_, err = db.Exec("VACUUM")
if err != nil {
return err
}**/
// 创建
err = this.initTables(db, 1)
if err != nil {
return err
}
// 读取总数量
row := this.db.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
if row.Err() != nil {
return row.Err()
}
var total int64
err = row.Scan(&total)
if err != nil {
return err
}
this.total = total
// 常用语句
this.existsByHashStmt, err = this.db.Prepare(`SELECT "bodySize" FROM "` + this.itemsTableName + `" WHERE "hash"=? AND expiredAt>? LIMIT 1`)
if err != nil {
return err
}
this.insertStmt, err = this.db.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
this.selectByHashStmt, err = this.db.Prepare(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? LIMIT 1`)
if err != nil {
return err
}
this.deleteByHashStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.statStmt, err = this.db.Prepare(`SELECT COUNT(*), IFNULL(SUM(headerSize+bodySize+metaSize), 0), IFNULL(SUM(headerSize+bodySize), 0) FROM "` + this.itemsTableName + `"`)
if err != nil {
return err
}
this.purgeStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" WHERE expiredAt<=? LIMIT ?`)
if err != nil {
return err
}
this.deleteAllStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemsTableName + `"`)
if err != nil {
return err
}
return nil
}
func (this *FileList) Reset() error {
// 不做任何事情
return nil
}
func (this *FileList) Add(hash string, item *Item) error {
if this.isClosed {
return nil
}
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.Host, item.ServerId, utils.UnixTime())
if err != nil {
return err
}
atomic.AddInt64(&this.total, 1)
if this.onAdd != nil {
this.onAdd(item)
}
return nil
}
func (this *FileList) Exist(hash string) (bool, error) {
if this.isClosed {
return false, nil
}
rows, err := this.existsByHashStmt.Query(hash, time.Now().Unix())
if err != nil {
return false, err
}
defer func() {
_ = rows.Close()
}()
if rows.Next() {
return true, nil
}
return false, nil
}
// CleanPrefix 清理某个前缀的缓存数据
func (this *FileList) CleanPrefix(prefix string) error {
if this.isClosed {
return nil
}
if len(prefix) == 0 {
return nil
}
var count = int64(10000)
for {
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0 WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+strconv.FormatInt(count, 10)+`)`, utils.UnixTime(), prefix)
if err != nil {
return err
}
affectedRows, err := result.RowsAffected()
if err != nil {
return err
}
if affectedRows < count {
return nil
}
}
}
func (this *FileList) Remove(hash string) error {
if this.isClosed {
return nil
}
row := this.selectByHashStmt.QueryRow(hash)
if row.Err() != nil {
return row.Err()
}
var item = &Item{Type: ItemTypeFile}
err := row.Scan(&item.Key, &item.HeaderSize, &item.BodySize, &item.MetaSize, &item.ExpiredAt)
if err != nil {
if err == sql.ErrNoRows {
return nil
}
return err
}
_, err = this.deleteByHashStmt.Exec(hash)
if err != nil {
return err
}
atomic.AddInt64(&this.total, -1)
if this.onRemove != nil {
this.onRemove(item)
}
return nil
}
// Purge 清理过期的缓存
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用
func (this *FileList) Purge(count int, callback func(hash string) error) error {
if this.isClosed {
return nil
}
if count <= 0 {
count = 1000
}
rows, err := this.purgeStmt.Query(time.Now().Unix(), count)
if err != nil {
return err
}
hashStrings := []string{}
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
_ = rows.Close()
return err
}
hashStrings = append(hashStrings, hash)
}
_ = rows.Close() // 不能使用defer防止读写冲突
// 不在 rows.Next() 循环中操作是为了避免死锁
for _, hash := range hashStrings {
err = this.Remove(hash)
if err != nil {
return err
}
err = callback(hash)
if err != nil {
return err
}
}
return nil
}
func (this *FileList) CleanAll() error {
if this.isClosed {
return nil
}
_, err := this.deleteAllStmt.Exec()
if err != nil {
return err
}
atomic.StoreInt64(&this.total, 0)
return nil
}
func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
if this.isClosed {
return &Stat{}, nil
}
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
row := this.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
stat := &Stat{}
err := row.Scan(&stat.Count, &stat.Size, &stat.ValueSize)
if err != nil {
return nil, err
}
return stat, nil
}
// Count 总数量
// 常用的方法,所以避免直接查询数据库
func (this *FileList) Count() (int64, error) {
return atomic.LoadInt64(&this.total), nil
}
// OnAdd 添加事件
func (this *FileList) OnAdd(f func(item *Item)) {
this.onAdd = f
}
// OnRemove 删除事件
func (this *FileList) OnRemove(f func(item *Item)) {
this.onRemove = f
}
func (this *FileList) Close() error {
this.isClosed = true
if this.db != nil {
_ = this.existsByHashStmt.Close()
_ = this.insertStmt.Close()
_ = this.selectByHashStmt.Close()
_ = this.deleteByHashStmt.Close()
_ = this.statStmt.Close()
_ = this.purgeStmt.Close()
_ = this.deleteAllStmt.Close()
return this.db.Close()
}
return nil
}
// 初始化
func (this *FileList) initTables(db *sql.DB, times int) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"key" varchar(1024),
"headerSize" integer DEFAULT 0,
"bodySize" integer DEFAULT 0,
"metaSize" integer DEFAULT 0,
"expiredAt" integer DEFAULT 0,
"createdAt" integer DEFAULT 0,
"host" varchar(128),
"serverId" integer
);
CREATE INDEX IF NOT EXISTS "createdAt"
ON "` + this.itemsTableName + `" (
"createdAt" ASC
);
CREATE INDEX IF NOT EXISTS "expiredAt"
ON "` + this.itemsTableName + `" (
"expiredAt" ASC
);
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
ON "` + this.itemsTableName + `" (
"hash" ASC
);
CREATE INDEX IF NOT EXISTS "serverId"
ON "` + this.itemsTableName + `" (
"serverId" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := db.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(db, times+1)
}
return err
}
return err
}
return nil
}
// 删除过期不用的表格
func (this *FileList) removeOldTables() error {
rows, err := this.db.Query(`SELECT "name" FROM sqlite_master WHERE "type"='table'`)
if err != nil {
return err
}
defer func() {
_ = rows.Close()
}()
for rows.Next() {
var name string
err = rows.Scan(&name)
if err != nil {
return err
}
if lists.ContainsString(this.oldTables, name) {
// 异步执行
go func() {
remotelogs.Println("CACHE", "remove old table '"+name+"' ...")
_, _ = this.db.Exec(`DROP TABLE "` + name + `"`)
remotelogs.Println("CACHE", "remove old table '"+name+"' done")
}()
}
}
return nil
}

View File

@@ -0,0 +1,269 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/rands"
stringutil "github.com/iwind/TeaGo/utils/string"
"strconv"
"sync"
"testing"
"time"
)
func TestFileList_Init(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestFileList_Add(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
err = list.Add(stringutil.Md5("123456"), &Item{
Key: "123456",
ExpiredAt: time.Now().Unix(),
HeaderSize: 1,
MetaSize: 2,
BodySize: 3,
Host: "teaos.cn",
ServerId: 1,
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestFileList_Add_Many(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 2000_0000; i++ {
u := "http://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &Item{
Key: u,
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1,
MetaSize: 2,
BodySize: 3,
})
if err != nil {
t.Fatal(err)
}
if i > 0 && i%10_000 == 0 {
t.Log(i, int(10000/time.Since(before).Seconds()), "qps")
before = time.Now()
}
}
t.Log("ok")
}
func TestFileList_Exist(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
{
exists, err := list.Exist(stringutil.Md5("123456"))
if err != nil {
t.Fatal(err)
}
t.Log("exists:", exists)
}
{
exists, err := list.Exist(stringutil.Md5("http://edge.teaos.cn/1234561"))
if err != nil {
t.Fatal(err)
}
t.Log("exists:", exists)
}
}
func TestFileList_Exist_Many_DB(t *testing.T) {
// 测试在多个数据库下的性能
var listSlice = []ListInterface{}
for i := 1; i <= 10; i++ {
list := NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
err := list.Init()
if err != nil {
t.Fatal(err)
}
listSlice = append(listSlice, list)
}
var wg = sync.WaitGroup{}
var threads = 8
wg.Add(threads)
var count = 200_000
var countLocker sync.Mutex
var tasks = make(chan int, count)
for i := 0; i < count; i++ {
tasks <- i
}
var hash = stringutil.Md5("http://edge.teaos.cn/1234561")
before := time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
for i := 0; i < threads; i++ {
go func() {
defer wg.Done()
for {
select {
case <-tasks:
countLocker.Lock()
count--
countLocker.Unlock()
var list = listSlice[rands.Int(0, len(listSlice)-1)]
_, _ = list.Exist(hash)
default:
return
}
}
}()
}
wg.Wait()
t.Log("left:", count)
}
func TestFileList_CleanPrefix(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
err = list.CleanPrefix("1234")
if err != nil {
t.Fatal(err)
}
t.Log(time.Since(before).Seconds()*1000, "ms")
}
func TestFileList_Remove(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
list.OnRemove(func(item *Item) {
t.Logf("remove %#v", item)
})
err = list.Remove(stringutil.Md5("123456"))
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestFileList_Purge(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
err = list.Purge(2, func(hash string) error {
t.Log(hash)
return nil
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestFileList_Stat(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
stat, err := list.Stat(nil)
if err != nil {
t.Fatal(err)
}
t.Log("count:", stat.Count, "size:", stat.Size, "valueSize:", stat.ValueSize)
}
func TestFileList_Count(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
count, err := list.Count()
if err != nil {
t.Fatal(err)
}
t.Log("count:", count)
t.Log(time.Since(before).Seconds()*1000, "ms")
}
func TestFileList_CleanAll(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
err = list.CleanAll()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
t.Log(list.Count())
}
func TestFileList_Conflict(t *testing.T) {
list := NewFileList(Tea.Root + "/data").(*FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
rows, err := list.purgeStmt.Query(time.Now().Unix(), 1000)
if err != nil {
t.Fatal(err)
}
go func() {
time.Sleep(5 * time.Second)
_ = rows.Close()
}()
t.Log("before exists")
t.Log(list.Exist("123456"))
t.Log("after exists")
}
func BenchmarkFileList_Exist(b *testing.B) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f")
}
}

View File

@@ -0,0 +1,44 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
type ListInterface interface {
// Init 初始化
Init() error
// Reset 重置数据
Reset() error
// Add 添加内容
Add(hash string, item *Item) error
// Exist 检查内容是否存在
Exist(hash string) (bool, error)
// CleanPrefix 清除某个前缀的缓存
CleanPrefix(prefix string) error
// Remove 删除内容
Remove(hash string) error
// Purge 清理过期数据
Purge(count int, callback func(hash string) error) error
// CleanAll 清除所有缓存
CleanAll() error
// Stat 统计
Stat(check func(hash string) bool) (*Stat, error)
// Count 总数量
Count() (int64, error)
// OnAdd 添加事件
OnAdd(f func(item *Item))
// OnRemove 删除事件
OnRemove(f func(item *Item))
// Close 关闭
Close() error
}

View File

@@ -0,0 +1,254 @@
package caches
import (
"github.com/iwind/TeaGo/logs"
"strconv"
"strings"
"sync"
"testing"
)
// MemoryList 内存缓存列表管理
type MemoryList struct {
itemMaps map[string]map[string]*Item // prefix => { hash => item }
prefixes []string
locker sync.RWMutex
onAdd func(item *Item)
onRemove func(item *Item)
purgeIndex int
}
func NewMemoryList() ListInterface {
return &MemoryList{
itemMaps: map[string]map[string]*Item{},
}
}
func (this *MemoryList) Init() error {
this.prefixes = []string{"000"}
for i := 100; i <= 999; i++ {
this.prefixes = append(this.prefixes, strconv.Itoa(i))
}
for _, prefix := range this.prefixes {
this.itemMaps[prefix] = map[string]*Item{}
}
return nil
}
func (this *MemoryList) Reset() error {
this.locker.Lock()
for key := range this.itemMaps {
this.itemMaps[key] = map[string]*Item{}
}
this.locker.Unlock()
return nil
}
func (this *MemoryList) Add(hash string, item *Item) error {
this.locker.Lock()
prefix := this.prefix(hash)
itemMap, ok := this.itemMaps[prefix]
if !ok {
itemMap = map[string]*Item{}
this.itemMaps[prefix] = itemMap
}
// 先删除,为了可以正确触发统计
oldItem, ok := itemMap[hash]
if ok {
if this.onRemove != nil {
this.onRemove(oldItem)
}
}
// 添加
if this.onAdd != nil {
this.onAdd(item)
}
itemMap[hash] = item
this.locker.Unlock()
return nil
}
func (this *MemoryList) Exist(hash string) (bool, error) {
this.locker.RLock()
defer this.locker.RUnlock()
prefix := this.prefix(hash)
itemMap, ok := this.itemMaps[prefix]
if !ok {
return false, nil
}
item, ok := itemMap[hash]
if !ok {
return false, nil
}
return !item.IsExpired(), nil
}
// CleanPrefix 根据前缀进行清除
func (this *MemoryList) CleanPrefix(prefix string) error {
this.locker.RLock()
defer this.locker.RUnlock()
// TODO 需要优化性能支持千万级数据低于1s的处理速度
for _, itemMap := range this.itemMaps {
for _, item := range itemMap {
if strings.HasPrefix(item.Key, prefix) {
item.ExpiredAt = 0
}
}
}
return nil
}
func (this *MemoryList) Remove(hash string) error {
this.locker.Lock()
itemMap, ok := this.itemMaps[this.prefix(hash)]
if !ok {
this.locker.Unlock()
return nil
}
item, ok := itemMap[hash]
if ok {
if this.onRemove != nil {
this.onRemove(item)
}
delete(itemMap, hash)
}
this.locker.Unlock()
return nil
}
// Purge 清理过期的缓存
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用
func (this *MemoryList) Purge(count int, callback func(hash string) error) error {
this.locker.Lock()
deletedHashList := []string{}
if this.purgeIndex >= len(this.prefixes) {
this.purgeIndex = 0
}
prefix := this.prefixes[this.purgeIndex]
this.purgeIndex++
itemMap, ok := this.itemMaps[prefix]
if !ok {
this.locker.Unlock()
return nil
}
for hash, item := range itemMap {
if count <= 0 {
break
}
if item.IsExpired() {
if this.onRemove != nil {
this.onRemove(item)
}
delete(itemMap, hash)
deletedHashList = append(deletedHashList, hash)
}
count--
}
this.locker.Unlock()
// 执行外部操作
for _, hash := range deletedHashList {
if callback != nil {
err := callback(hash)
if err != nil {
return err
}
}
}
return nil
}
func (this *MemoryList) CleanAll() error {
return this.Reset()
}
func (this *MemoryList) Stat(check func(hash string) bool) (*Stat, error) {
this.locker.RLock()
defer this.locker.RUnlock()
result := &Stat{
Count: 0,
Size: 0,
}
for _, itemMap := range this.itemMaps {
for hash, item := range itemMap {
if !item.IsExpired() {
// 检查文件是否存在、内容是否正确等
if check != nil && check(hash) {
result.Count++
result.ValueSize += item.Size()
result.Size += item.TotalSize()
}
}
}
}
return result, nil
}
// Count 总数量
func (this *MemoryList) Count() (int64, error) {
this.locker.RLock()
var count = 0
for _, itemMap := range this.itemMaps {
count += len(itemMap)
}
this.locker.RUnlock()
return int64(count), nil
}
// OnAdd 添加事件
func (this *MemoryList) OnAdd(f func(item *Item)) {
this.onAdd = f
}
// OnRemove 删除事件
func (this *MemoryList) OnRemove(f func(item *Item)) {
this.onRemove = f
}
func (this *MemoryList) Close() error {
return nil
}
func (this *MemoryList) print(t *testing.T) {
this.locker.Lock()
for _, itemMap := range this.itemMaps {
if len(itemMap) > 0 {
logs.PrintAsJSON(itemMap, t)
}
}
this.locker.Unlock()
}
func (this *MemoryList) prefix(hash string) string {
var prefix string
if len(hash) > 3 {
prefix = hash[:3]
} else {
prefix = hash
}
_, ok := this.itemMaps[prefix]
if !ok {
prefix = "000"
}
return prefix
}

View File

@@ -0,0 +1,190 @@
package caches
import (
"fmt"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/rands"
"math/rand"
"strconv"
"testing"
"time"
)
func TestMemoryList_Add(t *testing.T) {
list := NewMemoryList().(*MemoryList)
_ = list.Init()
_ = list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("123456", &Item{
Key: "c1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
t.Log(list.prefixes)
logs.PrintAsJSON(list.itemMaps, t)
}
func TestMemoryList_Remove(t *testing.T) {
list := NewMemoryList().(*MemoryList)
_ = list.Init()
_ = list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Remove("b")
list.print(t)
}
func TestMemoryList_Purge(t *testing.T) {
list := NewMemoryList().(*MemoryList)
_ = list.Init()
_ = list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("c", &Item{
Key: "c1",
ExpiredAt: time.Now().Unix() - 3600,
HeaderSize: 1024,
})
_ = list.Add("d", &Item{
Key: "d1",
ExpiredAt: time.Now().Unix() - 2,
HeaderSize: 1024,
})
_ = list.Purge(100, func(hash string) error {
t.Log("delete:", hash)
return nil
})
list.print(t)
for i := 0; i < 1000; i++ {
_ = list.Purge(100, func(hash string) error {
t.Log("delete:", hash)
return nil
})
t.Log(list.purgeIndex)
}
}
func TestMemoryList_Purge_Large_List(t *testing.T) {
list := NewMemoryList().(*MemoryList)
_ = list.Init()
for i := 0; i < 1_000_000; i++ {
_ = list.Add("a"+strconv.Itoa(i), &Item{
Key: "a" + strconv.Itoa(i),
ExpiredAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)),
HeaderSize: 1024,
})
}
time.Sleep(1 * time.Hour)
}
func TestMemoryList_Stat(t *testing.T) {
list := NewMemoryList()
_ = list.Init()
_ = list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
_ = list.Add("c", &Item{
Key: "c1",
ExpiredAt: time.Now().Unix(),
HeaderSize: 1024,
})
_ = list.Add("d", &Item{
Key: "d1",
ExpiredAt: time.Now().Unix() - 2,
HeaderSize: 1024,
})
result, _ := list.Stat(func(hash string) bool {
// 随机测试
rand.Seed(time.Now().UnixNano())
return rand.Int()%2 == 0
})
t.Log(result)
}
func TestMemoryList_CleanPrefix(t *testing.T) {
list := NewMemoryList()
_ = list.Init()
before := time.Now()
for i := 0; i < 1_000_000; i++ {
key := "http://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
Key: key,
ExpiredAt: time.Now().Unix() + 3600,
BodySize: 0,
HeaderSize: 0,
})
}
t.Log(time.Since(before).Seconds()*1000, "ms")
before = time.Now()
err := list.CleanPrefix("http://www.teaos.cn/hello/10")
if err != nil {
t.Fatal(err)
}
logs.Println(list.Stat(func(hash string) bool {
return true
}))
t.Log(time.Since(before).Seconds()*1000, "ms")
}
func TestMemoryList_GC(t *testing.T) {
list := NewMemoryList().(*MemoryList)
_ = list.Init()
for i := 0; i < 1_000_000; i++ {
key := "http://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
Key: key,
ExpiredAt: 0,
BodySize: 0,
HeaderSize: 0,
})
}
time.Sleep(10 * time.Second)
t.Log("clean...", len(list.itemMaps))
_ = list.CleanAll()
t.Log("cleanAll...", len(list.itemMaps))
before := time.Now()
//runtime.GC()
t.Log("gc cost:", time.Since(before).Seconds()*1000, "ms")
timeout := time.NewTimer(2 * time.Minute)
<-timeout.C
t.Log("2 minutes passed")
time.Sleep(30 * time.Minute)
}

View File

@@ -1,119 +0,0 @@
package caches
import (
"fmt"
"github.com/cespare/xxhash"
"math/rand"
"strconv"
"testing"
"time"
)
func TestList_Add(t *testing.T) {
list := NewList()
list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
t.Log(list.m)
}
func TestList_Remove(t *testing.T) {
list := NewList()
list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Remove("b")
t.Log(list.m)
}
func TestList_Purge(t *testing.T) {
list := NewList()
list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Add("c", &Item{
Key: "c1",
ExpiredAt: time.Now().Unix() - 3600,
HeaderSize: 1024,
})
list.Add("d", &Item{
Key: "d1",
ExpiredAt: time.Now().Unix() - 2,
HeaderSize: 1024,
})
list.Purge(100, func(hash string) {
t.Log("delete:", hash)
})
t.Log(list.m)
}
func TestList_Stat(t *testing.T) {
list := NewList()
list.Add("a", &Item{
Key: "a1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Add("b", &Item{
Key: "b1",
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1024,
})
list.Add("c", &Item{
Key: "c1",
ExpiredAt: time.Now().Unix(),
HeaderSize: 1024,
})
list.Add("d", &Item{
Key: "d1",
ExpiredAt: time.Now().Unix() - 2,
HeaderSize: 1024,
})
result := list.Stat(func(hash string) bool {
// 随机测试
rand.Seed(time.Now().UnixNano())
return rand.Int()%2 == 0
})
t.Log(result)
}
func TestList_FindKeysWithPrefix(t *testing.T) {
list := NewList()
before := time.Now()
for i := 0; i < 1_000_000; i++ {
key := "http://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
Key: key,
ExpiredAt: 0,
BodySize: 0,
HeaderSize: 0,
})
}
t.Log(time.Since(before).Seconds()*1000, "ms")
before = time.Now()
keys := list.FindKeysWithPrefix("http://www.teaos.cn/hello/5000")
t.Log(len(keys))
t.Log(time.Since(before).Seconds()*1000, "ms")
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"strconv"
"sync"
)
@@ -162,3 +163,25 @@ func (this *Manager) TotalMemorySize() int64 {
}
return total
}
// FindAllCachePaths 所有缓存路径
func (this *Manager) FindAllCachePaths() []string {
this.locker.Lock()
defer this.locker.Unlock()
var result = []string{}
for _, policy := range this.policyMap {
if policy.Type == serverconfigs.CachePolicyStorageFile {
if policy.Options != nil {
dir, ok := policy.Options["dir"]
if ok {
var dirString = types.String(dir)
if len(dirString) > 0 {
result = append(result, dirString)
}
}
}
}
}
return result
}

View File

@@ -3,27 +3,33 @@ package caches
type ReaderFunc func(n int) (goNext bool, err error)
type Reader interface {
// 初始化
// Init 初始化
Init() error
// 状态码
// TypeName 类型名称
TypeName() string
// Status 状态码
Status() int
// 读取Header
// LastModified 最后修改时间
LastModified() int64
// ReadHeader 读取Header
ReadHeader(buf []byte, callback ReaderFunc) error
// 读取Body
// ReadBody 读取Body
ReadBody(buf []byte, callback ReaderFunc) error
// 读取某个范围内的Body
// ReadBodyRange 读取某个范围内的Body
ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error
// Header Size
// HeaderSize Header Size
HeaderSize() int64
// Body Size
// BodySize Body Size
BodySize() int64
// 关闭
// Close 关闭
Close() error
}

View File

@@ -106,10 +106,22 @@ func (this *FileReader) Init() error {
return nil
}
func (this *FileReader) TypeName() string {
return "disk"
}
func (this *FileReader) Status() int {
return this.status
}
func (this *FileReader) LastModified() int64 {
stat, err := this.fp.Stat()
if err != nil {
return 0
}
return stat.ModTime().Unix()
}
func (this *FileReader) HeaderSize() int64 {
return int64(this.headerSize)
}

View File

@@ -16,10 +16,18 @@ func (this *MemoryReader) Init() error {
return nil
}
func (this *MemoryReader) TypeName() string {
return "memory"
}
func (this *MemoryReader) Status() int {
return this.item.Status
}
func (this *MemoryReader) LastModified() int64 {
return this.item.ModifiedAt
}
func (this *MemoryReader) HeaderSize() int64 {
return int64(len(this.item.HeaderValue))
}

View File

@@ -12,6 +12,8 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
stringutil "github.com/iwind/TeaGo/utils/string"
"golang.org/x/text/language"
"golang.org/x/text/message"
"io"
"os"
"path/filepath"
@@ -34,12 +36,6 @@ const (
SizeMeta = SizeExpiresAt + SizeStatus + SizeURLLength + SizeHeaderLength + SizeBodyLength
)
var (
ErrNotFound = errors.New("cache not found")
ErrFileIsWriting = errors.New("the file is writing")
ErrInvalidRange = errors.New("invalid range")
)
// FileStorage 文件缓存
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
@@ -49,15 +45,16 @@ type FileStorage struct {
memoryStorage *MemoryStorage // 一级缓存
totalSize int64
list *List
locker sync.RWMutex
ticker *utils.Ticker
list ListInterface
writingKeyMap map[string]bool // key => bool
locker sync.RWMutex
ticker *utils.Ticker
}
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
return &FileStorage{
policy: policy,
list: NewList(),
policy: policy,
writingKeyMap: map[string]bool{},
}
}
@@ -68,41 +65,10 @@ func (this *FileStorage) Policy() *serverconfigs.HTTPCachePolicy {
// Init 初始化
func (this *FileStorage) Init() error {
this.list.OnAdd(func(item *Item) {
atomic.AddInt64(&this.totalSize, item.TotalSize())
})
this.list.OnRemove(func(item *Item) {
atomic.AddInt64(&this.totalSize, -item.TotalSize())
})
this.locker.Lock()
defer this.locker.Unlock()
before := time.Now()
cacheDir := ""
defer func() {
// 统计
count := 0
size := int64(0)
if this.list != nil {
stat := this.list.Stat(func(hash string) bool {
return true
})
count = stat.Count
size = stat.Size
}
cost := time.Since(before).Seconds() * 1000
sizeMB := strconv.FormatInt(size, 10) + " Bytes"
if size > 1024*1024*1024 {
sizeMB = fmt.Sprintf("%.3f G", float64(size)/1024/1024/1024)
} else if size > 1024*1024 {
sizeMB = fmt.Sprintf("%.3f M", float64(size)/1024/1024)
} else if size > 1024 {
sizeMB = fmt.Sprintf("%.3f K", float64(size)/1024)
}
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+cacheDir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+strconv.Itoa(count)+", size: "+sizeMB)
}()
// 配置
cacheConfig := &serverconfigs.HTTPFileCacheStorage{}
@@ -115,7 +81,7 @@ func (this *FileStorage) Init() error {
return err
}
this.cacheConfig = cacheConfig
cacheDir = cacheConfig.Dir
cacheDir := cacheConfig.Dir
if !filepath.IsAbs(this.cacheConfig.Dir) {
this.cacheConfig.Dir = Tea.Root + Tea.DS + this.cacheConfig.Dir
@@ -127,6 +93,26 @@ func (this *FileStorage) Init() error {
return errors.New("[CACHE]cache storage dir can not be empty")
}
list := NewFileList(dir + "/p" + strconv.FormatInt(this.policy.Id, 10))
err = list.Init()
if err != nil {
return err
}
this.list = list
stat, err := list.Stat(func(hash string) bool {
return true
})
if err != nil {
return err
}
this.totalSize = stat.Size
this.list.OnAdd(func(item *Item) {
atomic.AddInt64(&this.totalSize, item.TotalSize())
})
this.list.OnRemove(func(item *Item) {
atomic.AddInt64(&this.totalSize, -item.TotalSize())
})
// 检查目录是否存在
_, err = os.Stat(dir)
if err != nil {
@@ -140,6 +126,23 @@ func (this *FileStorage) Init() error {
}
}
defer func() {
// 统计
count := stat.Count
size := stat.Size
cost := time.Since(before).Seconds() * 1000
sizeMB := strconv.FormatInt(size, 10) + " Bytes"
if size > 1024*1024*1024 {
sizeMB = fmt.Sprintf("%.3f G", float64(size)/1024/1024/1024)
} else if size > 1024*1024 {
sizeMB = fmt.Sprintf("%.3f M", float64(size)/1024/1024)
} else if size > 1024 {
sizeMB = fmt.Sprintf("%.3f K", float64(size)/1024)
}
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+cacheDir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
}()
// 初始化list
err = this.initList()
if err != nil {
@@ -189,11 +192,9 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
}
hash, path := this.keyPath(key)
if !this.list.Exist(hash) {
return nil, ErrNotFound
}
// TODO 尝试使用mmap加快读取速度
var isOk = false
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
if err != nil {
if !os.IsNotExist(err) {
@@ -201,6 +202,21 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
}
return nil, ErrNotFound
}
defer func() {
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
// 检查文件记录是否已过期
exists, err := this.list.Exist(hash)
if err != nil {
return nil, err
}
if !exists {
return nil, ErrNotFound
}
reader := NewFileReader(fp)
if err != nil {
@@ -210,6 +226,8 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
if err != nil {
return nil, err
}
isOk = true
return reader, nil
}
@@ -223,18 +241,41 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
}
}
// 是否正在写入
var isWriting = false
this.locker.Lock()
_, ok := this.writingKeyMap[key]
this.locker.Unlock()
if ok {
return nil, ErrFileIsWriting
}
this.locker.Lock()
this.writingKeyMap[key] = true
this.locker.Unlock()
defer func() {
if !isWriting {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
}
}()
// 检查是否超出最大值
if this.policy.MaxKeys > 0 && this.list.Count() > this.policy.MaxKeys {
return nil, errors.New("write file cache failed: too many keys in cache storage")
count, err := this.list.Count()
if err != nil {
return nil, err
}
if this.policy.MaxKeys > 0 && count > this.policy.MaxKeys {
return nil, NewCapacityError("write file cache failed: too many keys in cache storage")
}
capacityBytes := this.diskCapacityBytes()
if capacityBytes > 0 && capacityBytes <= this.totalSize {
return nil, errors.New("write file cache failed: over disk size, real size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
return nil, NewCapacityError("write file cache failed: over disk size, current total size: " + strconv.FormatInt(this.totalSize, 10) + " bytes, capacity: " + strconv.FormatInt(capacityBytes, 10))
}
hash := stringutil.Md5(key)
dir := this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
_, err := os.Stat(dir)
_, err = os.Stat(dir)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
@@ -246,13 +287,17 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
}
// 先删除
this.list.Remove(hash)
err = this.list.Remove(hash)
if err != nil {
return nil, err
}
path := dir + "/" + hash + ".cache.tmp"
writer, err := os.OpenFile(path, os.O_CREATE|os.O_SYNC|os.O_WRONLY, 0666)
if err != nil {
return nil, err
}
isWriting = true
isOk := false
removeOnFailure := true
@@ -337,7 +382,11 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
isOk = true
return NewFileWriter(writer, key, expiredAt), nil
return NewFileWriter(writer, key, expiredAt, func() {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
}), nil
}
// AddToList 添加到List
@@ -351,7 +400,10 @@ func (this *FileStorage) AddToList(item *Item) {
item.MetaSize = SizeMeta
hash := stringutil.Md5(item.Key)
this.list.Add(hash, item)
err := this.list.Add(hash, item)
if err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
remotelogs.Error("CACHE", "add to list failed: "+err.Error())
}
}
// Delete 删除某个键值对应的缓存
@@ -365,8 +417,11 @@ func (this *FileStorage) Delete(key string) error {
}
hash, path := this.keyPath(key)
this.list.Remove(hash)
err := os.Remove(path)
err := this.list.Remove(hash)
if err != nil {
return err
}
err = os.Remove(path)
if err == nil || os.IsNotExist(err) {
return nil
}
@@ -380,7 +435,7 @@ func (this *FileStorage) Stat() (*Stat, error) {
return this.list.Stat(func(hash string) bool {
return true
}), nil
})
}
// CleanAll 清除所有的缓存
@@ -393,7 +448,10 @@ func (this *FileStorage) CleanAll() error {
_ = this.memoryStorage.CleanAll()
}
this.list.Reset()
err := this.list.CleanAll()
if err != nil {
return err
}
// 删除缓存和目录
// 不能直接删除子目录,比较危险
@@ -415,6 +473,7 @@ func (this *FileStorage) CleanAll() error {
return nil
}
// 改成待删除
subDirs, err := fp.Readdir(-1)
if err != nil {
return err
@@ -431,13 +490,22 @@ func (this *FileStorage) CleanAll() error {
continue
}
// 删除目录
err = os.RemoveAll(dir + "/" + subDir)
// 修改目录
tmpDir := dir + "/" + subDir + "-deleted"
err = os.Rename(dir+"/"+subDir, tmpDir)
if err != nil {
return err
}
}
// 重新遍历待删除
go func() {
err = this.cleanDeletedDirs(dir)
if err != nil {
remotelogs.Warn("CACHE", "delete '*-deleted' dirs failed: "+err.Error())
}
}()
return nil
}
@@ -453,29 +521,25 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
// 目录
if urlType == "dir" {
resultKeys := []string{}
for _, key := range keys {
resultKeys = append(resultKeys, this.list.FindKeysWithPrefix(key)...)
err := this.list.CleanPrefix(key)
if err != nil {
return err
}
}
keys = resultKeys
}
// 文件
for _, key := range keys {
hash, path := this.keyPath(key)
if !this.list.Exist(hash) {
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
return err
}
continue
}
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
return err
}
this.list.Remove(hash)
err = this.list.Remove(hash)
if err != nil {
return err
}
}
return nil
}
@@ -490,10 +554,12 @@ func (this *FileStorage) Stop() {
this.memoryStorage.Stop()
}
this.list.Reset()
_ = this.list.Reset()
if this.ticker != nil {
this.ticker.Stop()
}
_ = this.list.Close()
}
// TotalDiskSize 消耗的磁盘尺寸
@@ -534,45 +600,23 @@ func (this *FileStorage) hashPath(hash string) (path string) {
// 初始化List
func (this *FileStorage) initList() error {
this.list.Reset()
dir := this.dir()
// 清除tmp
files, err := filepath.Glob(dir + "/*/*/*.cache.tmp")
err := this.list.Reset()
if err != nil {
return err
}
for _, path := range files {
_ = os.Remove(path)
}
// 加载缓存
files, err = filepath.Glob(dir + "/*/*/*.cache")
if err != nil {
return err
}
for _, path := range files {
basename := filepath.Base(path)
index := strings.LastIndex(basename, ".")
if index < 0 {
continue
}
hash := basename[:index]
// 使用异步防止阻塞主线程
go func() {
dir := this.dir()
// 解析文件信息
item, err := this.decodeFile(path)
if err != nil {
if err != ErrNotFound {
remotelogs.Error("CACHE", "decode path '"+path+"': "+err.Error())
// 清除tmp
files, err := filepath.Glob(dir + "/*/*/*.cache.tmp")
if err == nil && len(files) > 0 {
for _, path := range files {
_ = os.Remove(path)
}
continue
}
if item == nil {
continue
}
this.list.Add(hash, item)
}
}()
// 启动定时清理任务
this.ticker = utils.NewTicker(30 * time.Second)
@@ -686,13 +730,17 @@ func (this *FileStorage) decodeFile(path string) (*Item, error) {
// 清理任务
func (this *FileStorage) purgeLoop() {
this.list.Purge(1000, func(hash string) {
err := this.list.Purge(1000, func(hash string) error {
path := this.hashPath(hash)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge file storage failed: "+err.Error())
}
}
func (this *FileStorage) readToBuff(fp *os.File, buf []byte) (ok bool, err error) {
@@ -733,3 +781,34 @@ func (this *FileStorage) diskCapacityBytes() int64 {
}
return c1
}
// 清理 *-deleted 目录
// 由于在很多硬盘上耗时非常久,所以应该放在后台运行
func (this *FileStorage) cleanDeletedDirs(dir string) error {
fp, err := os.Open(dir)
if err != nil {
return err
}
defer func() {
_ = fp.Close()
}()
subDirs, err := fp.Readdir(-1)
if err != nil {
return err
}
for _, info := range subDirs {
subDir := info.Name()
if !strings.HasSuffix(subDir, "-deleted") {
continue
}
// 删除
err = os.RemoveAll(dir + "/" + subDir)
if err != nil {
if !os.IsNotExist(err) {
return err
}
}
}
return nil
}

View File

@@ -40,7 +40,7 @@ func TestFileStorage_Init(t *testing.T) {
time.Sleep(2 * time.Second)
storage.purgeLoop()
t.Log(len(storage.list.m), "entries left")
t.Log(storage.list.(*FileList).total, "entries left")
}
func TestFileStorage_OpenWriter(t *testing.T) {
@@ -441,14 +441,16 @@ func TestFileStorage_CleanAll(t *testing.T) {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
t.Log("before:", storage.list.m)
c, _ := storage.list.Count()
t.Log("before:", c)
err = storage.CleanAll()
if err != nil {
t.Fatal(err)
}
t.Log("after:", storage.list.m)
c, _ = storage.list.Count()
t.Log("after:", c)
t.Log("ok")
}

View File

@@ -3,7 +3,7 @@ package caches
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/cespare/xxhash"
"strconv"
@@ -18,38 +18,47 @@ type MemoryItem struct {
BodyValue []byte
Status int
IsDone bool
ModifiedAt int64
}
func (this *MemoryItem) IsExpired() bool {
return this.ExpiredAt < utils.UnixTime()
}
type MemoryStorage struct {
policy *serverconfigs.HTTPCachePolicy
list *List
list ListInterface
locker *sync.RWMutex
valuesMap map[uint64]*MemoryItem
ticker *utils.Ticker
purgeDuration time.Duration
totalSize int64
writingKeyMap map[string]bool // key => bool
}
func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy) *MemoryStorage {
return &MemoryStorage{
policy: policy,
list: NewList(),
locker: &sync.RWMutex{},
valuesMap: map[uint64]*MemoryItem{},
policy: policy,
list: NewMemoryList(),
locker: &sync.RWMutex{},
valuesMap: map[uint64]*MemoryItem{},
writingKeyMap: map[string]bool{},
}
}
// Init 初始化
func (this *MemoryStorage) Init() error {
_ = this.list.Init()
this.list.OnAdd(func(item *Item) {
atomic.AddInt64(&this.totalSize, item.Size())
atomic.AddInt64(&this.totalSize, item.TotalSize())
})
this.list.OnRemove(func(item *Item) {
atomic.AddInt64(&this.totalSize, -item.Size())
atomic.AddInt64(&this.totalSize, -item.TotalSize())
})
if this.purgeDuration <= 0 {
this.purgeDuration = 30 * time.Second
this.purgeDuration = 10 * time.Second
}
// 启动定时清理任务
@@ -68,10 +77,9 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
hash := this.hash(key)
this.locker.RLock()
defer this.locker.RUnlock()
item := this.valuesMap[hash]
if item == nil || !item.IsDone {
this.locker.RUnlock()
return nil, ErrNotFound
}
@@ -79,10 +87,13 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
reader := NewMemoryReader(item)
err := reader.Init()
if err != nil {
this.locker.RUnlock()
return nil, err
}
this.locker.RUnlock()
return reader, nil
}
this.locker.RUnlock()
_ = this.Delete(key)
@@ -91,22 +102,54 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
// OpenWriter 打开缓存写入器等待写入
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int) (Writer, error) {
this.locker.Lock()
defer this.locker.Unlock()
// 是否正在写入
var isWriting = false
_, ok := this.writingKeyMap[key]
if ok {
return nil, ErrFileIsWriting
}
this.writingKeyMap[key] = true
defer func() {
if !isWriting {
delete(this.writingKeyMap, key)
}
}()
// 检查是否过期
hash := this.hash(key)
item, ok := this.valuesMap[hash]
if ok && !item.IsExpired() {
return nil, ErrFileIsWriting
}
// 检查是否超出最大值
if this.policy.MaxKeys > 0 && this.list.Count() > this.policy.MaxKeys {
return nil, errors.New("write memory cache failed: too many keys in cache storage")
totalKeys, err := this.list.Count()
if err != nil {
return nil, err
}
if this.policy.MaxKeys > 0 && totalKeys > this.policy.MaxKeys {
return nil, NewCapacityError("write memory cache failed: too many keys in cache storage")
}
capacityBytes := this.memoryCapacityBytes()
if capacityBytes > 0 && capacityBytes <= this.totalSize {
return nil, errors.New("write memory cache failed: over memory size, real size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
}
// 先删除
err := this.Delete(key)
err = this.deleteWithoutLocker(key)
if err != nil {
return nil, err
}
return NewMemoryWriter(this.valuesMap, key, expiredAt, status, this.locker), nil
isWriting = true
return NewMemoryWriter(this.valuesMap, key, expiredAt, status, this.locker, func() {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
}), nil
}
// Delete 删除某个键值对应的缓存
@@ -114,7 +157,7 @@ func (this *MemoryStorage) Delete(key string) error {
hash := this.hash(key)
this.locker.Lock()
delete(this.valuesMap, hash)
this.list.Remove(fmt.Sprintf("%d", hash))
_ = this.list.Remove(fmt.Sprintf("%d", hash))
this.locker.Unlock()
return nil
}
@@ -126,14 +169,14 @@ func (this *MemoryStorage) Stat() (*Stat, error) {
return this.list.Stat(func(hash string) bool {
return true
}), nil
})
}
// CleanAll 清除所有缓存
func (this *MemoryStorage) CleanAll() error {
this.locker.Lock()
this.valuesMap = map[uint64]*MemoryItem{}
this.list.Reset()
_ = this.list.Reset()
atomic.StoreInt64(&this.totalSize, 0)
this.locker.Unlock()
return nil
@@ -143,11 +186,12 @@ func (this *MemoryStorage) CleanAll() error {
func (this *MemoryStorage) Purge(keys []string, urlType string) error {
// 目录
if urlType == "dir" {
resultKeys := []string{}
for _, key := range keys {
resultKeys = append(resultKeys, this.list.FindKeysWithPrefix(key)...)
err := this.list.CleanPrefix(key)
if err != nil {
return err
}
}
keys = resultKeys
}
for _, key := range keys {
@@ -162,13 +206,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
// Stop 停止缓存策略
func (this *MemoryStorage) Stop() {
this.locker.Lock()
defer this.locker.Unlock()
this.valuesMap = map[uint64]*MemoryItem{}
this.list.Reset()
this.writingKeyMap = map[string]bool{}
_ = this.list.Reset()
if this.ticker != nil {
this.ticker.Stop()
}
_ = this.list.Close()
this.locker.Unlock()
remotelogs.Println("CACHE", "close memory storage '"+strconv.FormatInt(this.policy.Id, 10)+"'")
}
// Policy 获取当前存储的Policy
@@ -180,7 +230,7 @@ func (this *MemoryStorage) Policy() *serverconfigs.HTTPCachePolicy {
func (this *MemoryStorage) AddToList(item *Item) {
item.MetaSize = int64(len(item.Key)) + 32 /** 32是我们评估的数据结构的长度 **/
hash := fmt.Sprintf("%d", this.hash(item.Key))
this.list.Add(hash, item)
_ = this.list.Add(hash, item)
}
// TotalDiskSize 消耗的磁盘尺寸
@@ -200,13 +250,14 @@ func (this *MemoryStorage) hash(key string) uint64 {
// 清理任务
func (this *MemoryStorage) purgeLoop() {
this.list.Purge(2048, func(hash string) {
_ = this.list.Purge(2048, func(hash string) error {
uintHash, err := strconv.ParseUint(hash, 10, 64)
if err == nil {
this.locker.Lock()
delete(this.valuesMap, uintHash)
this.locker.Unlock()
}
return nil
})
}
@@ -226,3 +277,10 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 {
}
return c1
}
func (this *MemoryStorage) deleteWithoutLocker(key string) error {
hash := this.hash(key)
delete(this.valuesMap, hash)
_ = this.list.Remove(fmt.Sprintf("%d", hash))
return nil
}

View File

@@ -84,6 +84,19 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
}
}
func TestMemoryStorage_OpenReaderLock(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
_ = storage.Init()
var h = storage.hash("test")
storage.valuesMap = map[uint64]*MemoryItem{
h: {
IsDone: true,
},
}
_, _ = storage.OpenReader("test")
}
func TestMemoryStorage_Delete(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
{
@@ -174,7 +187,8 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Log(storage.list.Count(), len(storage.valuesMap))
total, _ := storage.list.Count()
t.Log(total, len(storage.valuesMap))
}
func TestMemoryStorage_Purge(t *testing.T) {
@@ -208,7 +222,8 @@ func TestMemoryStorage_Purge(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Log(storage.list.Count(), len(storage.valuesMap))
total, _ := storage.list.Count()
t.Log(total, len(storage.valuesMap))
}
func TestMemoryStorage_Expire(t *testing.T) {
@@ -235,3 +250,18 @@ func TestMemoryStorage_Expire(t *testing.T) {
}
time.Sleep(70 * time.Second)
}
func TestMemoryStorage_Locker(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
err := storage.Init()
if err != nil {
t.Fatal(err)
}
storage.locker.Lock()
err = storage.deleteWithoutLocker("a")
storage.locker.Unlock()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -1,31 +1,31 @@
package caches
// 缓存内容写入接口
// Writer 缓存内容写入接口
type Writer interface {
// 写入Header数据
// WriteHeader 写入Header数据
WriteHeader(data []byte) (n int, err error)
// 写入Body数据
// Write 写入Body数据
Write(data []byte) (n int, err error)
// 写入的Header数据大小
// HeaderSize 写入的Header数据大小
HeaderSize() int64
// 写入的Body数据大小
// BodySize 写入的Body数据大小
BodySize() int64
// 关闭
// Close 关闭
Close() error
// 丢弃
// Discard 丢弃
Discard() error
// Key
// Key Key
Key() string
// 过期时间
// ExpiredAt 过期时间
ExpiredAt() int64
// 内容类型
// ItemType 内容类型
ItemType() ItemType
}

View File

@@ -14,17 +14,19 @@ type FileWriter struct {
headerSize int64
bodySize int64
expiredAt int64
endFunc func()
}
func NewFileWriter(rawWriter *os.File, key string, expiredAt int64) *FileWriter {
func NewFileWriter(rawWriter *os.File, key string, expiredAt int64, endFunc func()) *FileWriter {
return &FileWriter{
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
endFunc: endFunc,
}
}
// 写入数据
// WriteHeader 写入数据
func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
n, err = this.rawWriter.Write(data)
this.headerSize += int64(n)
@@ -34,7 +36,7 @@ func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
return
}
// 写入Header长度数据
// WriteHeaderLength 写入Header长度数据
func (this *FileWriter) WriteHeaderLength(headerLength int) error {
bytes4 := make([]byte, 4)
binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
@@ -51,7 +53,7 @@ func (this *FileWriter) WriteHeaderLength(headerLength int) error {
return nil
}
// 写入数据
// Write 写入数据
func (this *FileWriter) Write(data []byte) (n int, err error) {
n, err = this.rawWriter.Write(data)
this.bodySize += int64(n)
@@ -61,7 +63,7 @@ func (this *FileWriter) Write(data []byte) (n int, err error) {
return
}
// 写入Body长度数据
// WriteBodyLength 写入Body长度数据
func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
bytes8 := make([]byte, 8)
binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
@@ -78,8 +80,10 @@ func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
return nil
}
// 关闭
// Close 关闭
func (this *FileWriter) Close() error {
defer this.endFunc()
err := this.WriteHeaderLength(types.Int(this.headerSize))
if err != nil {
return err
@@ -103,8 +107,10 @@ func (this *FileWriter) Close() error {
return err
}
// 丢弃
// Discard 丢弃
func (this *FileWriter) Discard() error {
defer this.endFunc()
_ = this.rawWriter.Close()
err := os.Remove(this.rawWriter.Name())
@@ -127,7 +133,7 @@ func (this *FileWriter) Key() string {
return this.key
}
// 内容类型
// ItemType 获取内容类型
func (this *FileWriter) ItemType() ItemType {
return ItemTypeFile
}

View File

@@ -3,6 +3,7 @@ package caches
import (
"github.com/cespare/xxhash"
"sync"
"time"
)
type MemoryWriter struct {
@@ -14,52 +15,59 @@ type MemoryWriter struct {
bodySize int64
status int
hash uint64
item *MemoryItem
hash uint64
item *MemoryItem
endFunc func()
}
func NewMemoryWriter(m map[uint64]*MemoryItem, key string, expiredAt int64, status int, locker *sync.RWMutex) *MemoryWriter {
func NewMemoryWriter(m map[uint64]*MemoryItem, key string, expiredAt int64, status int, locker *sync.RWMutex, endFunc func()) *MemoryWriter {
w := &MemoryWriter{
m: m,
key: key,
expiredAt: expiredAt,
locker: locker,
item: &MemoryItem{
ExpiredAt: expiredAt,
Status: status,
ExpiredAt: expiredAt,
ModifiedAt: time.Now().Unix(),
Status: status,
},
status: status,
status: status,
endFunc: endFunc,
}
w.hash = w.calculateHash(key)
return w
}
// 写入数据
// WriteHeader 写入数据
func (this *MemoryWriter) WriteHeader(data []byte) (n int, err error) {
this.headerSize += int64(len(data))
this.item.HeaderValue = append(this.item.HeaderValue, data...)
return len(data), nil
}
// 写入数据
// Write 写入数据
func (this *MemoryWriter) Write(data []byte) (n int, err error) {
this.bodySize += int64(len(data))
this.item.BodyValue = append(this.item.BodyValue, data...)
return len(data), nil
}
// 数据尺寸
// HeaderSize 数据尺寸
func (this *MemoryWriter) HeaderSize() int64 {
return this.headerSize
}
// BodySize 主体内容尺寸
func (this *MemoryWriter) BodySize() int64 {
return this.bodySize
}
// 关闭
// Close 关闭
func (this *MemoryWriter) Close() error {
// 需要在Locker之外
defer this.endFunc()
if this.item == nil {
return nil
}
@@ -72,25 +80,28 @@ func (this *MemoryWriter) Close() error {
return nil
}
// 丢弃
// Discard 丢弃
func (this *MemoryWriter) Discard() error {
// 需要在Locker之外
defer this.endFunc()
this.locker.Lock()
delete(this.m, this.hash)
this.locker.Unlock()
return nil
}
// Key
// Key 获取Key
func (this *MemoryWriter) Key() string {
return this.key
}
// 过期时间
// ExpiredAt 过期时间
func (this *MemoryWriter) ExpiredAt() int64 {
return this.expiredAt
}
// 内容类型
// ItemType 内容类型
func (this *MemoryWriter) ItemType() ItemType {
return ItemTypeMemory
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.1.0"
Version = "0.2.5"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -6,7 +6,7 @@ import (
"sync"
)
// IP名单
// IPList IP名单
type IPList struct {
itemsMap map[int64]*IPItem // id => item
ipMap map[uint64][]int64 // ip => itemIds
@@ -96,7 +96,7 @@ func (this *IPList) Delete(itemId int64) {
this.isAll = len(this.ipMap[0]) > 0
}
// 判断是否包含某个IP
// Contains 判断是否包含某个IP
func (this *IPList) Contains(ip uint64) bool {
this.locker.RLock()
if this.isAll {
@@ -109,7 +109,7 @@ func (this *IPList) Contains(ip uint64) bool {
return ok
}
// 是否包含一组IP
// ContainsIPStrings 是否包含一组IP
func (this *IPList) ContainsIPStrings(ipStrings []string) (found bool, item *IPItem) {
if len(ipStrings) == 0 {
return

View File

@@ -5,6 +5,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/lionsoul2014/ip2region/binding/golang/ip2region"
"net"
"strings"
)
@@ -27,6 +28,9 @@ func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
if this.db == nil {
return nil, errors.New("library has not been loaded")

View File

@@ -47,7 +47,7 @@ func (this *IPListManager) Start() {
// 第一次读取
err := this.loop()
if err != nil {
remotelogs.Println("IP_LIST_MANAGER", err.Error())
remotelogs.Error("IP_LIST_MANAGER", err.Error())
}
ticker := time.NewTicker(60 * time.Second)
@@ -64,7 +64,7 @@ func (this *IPListManager) Start() {
if err != nil {
countErrors++
remotelogs.Println("IP_LIST_MANAGER", err.Error())
remotelogs.Error("IP_LIST_MANAGER", err.Error())
// 连续错误小于3次的我们立即重试
if countErrors <= 3 {

121
internal/metrics/manager.go Normal file
View File

@@ -0,0 +1,121 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package metrics
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"strconv"
"sync"
)
var SharedManager = NewManager()
type Manager struct {
tasks map[int64]*Task // itemId => *Task
categoryTasks map[string][]*Task // category => []*Task
locker sync.RWMutex
hasHTTPMetrics bool
hasTCPMetrics bool
hasUDPMetrics bool
}
func NewManager() *Manager {
return &Manager{
tasks: map[int64]*Task{},
categoryTasks: map[string][]*Task{},
}
}
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
this.locker.Lock()
defer this.locker.Unlock()
var newMap = map[int64]*serverconfigs.MetricItemConfig{}
for _, item := range items {
newMap[item.Id] = item
}
// 停用以前的 或 修改现在的
for itemId, task := range this.tasks {
newItem, ok := newMap[itemId]
if !ok || !newItem.IsOn { // 停用以前的
remotelogs.Println("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"'")
err := task.Stop()
if err != nil {
remotelogs.Error("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"' failed: "+err.Error())
}
delete(this.tasks, itemId)
} else { // 更新已存在的
if newItem.Version != task.item.Version {
remotelogs.Println("METRIC_MANAGER", "update task '"+strconv.FormatInt(itemId, 10)+"'")
task.item = newItem
}
}
}
// 启动新的
for _, newItem := range items {
if !newItem.IsOn {
continue
}
_, ok := this.tasks[newItem.Id]
if !ok {
remotelogs.Println("METRIC_MANAGER", "start task '"+strconv.FormatInt(newItem.Id, 10)+"'")
task := NewTask(newItem)
err := task.Init()
if err != nil {
remotelogs.Error("METRIC_MANAGER", "initialized task failed: "+err.Error())
continue
}
err = task.Start()
if err != nil {
remotelogs.Error("METRIC_MANAGER", "start task failed: "+err.Error())
continue
}
this.tasks[newItem.Id] = task
}
}
// 按分类存放
this.hasHTTPMetrics = false
this.hasTCPMetrics = false
this.hasUDPMetrics = false
this.categoryTasks = map[string][]*Task{}
for _, task := range this.tasks {
tasks := this.categoryTasks[task.item.Category]
tasks = append(tasks, task)
this.categoryTasks[task.item.Category] = tasks
switch task.item.Category {
case serverconfigs.MetricItemCategoryHTTP:
this.hasHTTPMetrics = true
case serverconfigs.MetricItemCategoryTCP:
this.hasTCPMetrics = true
case serverconfigs.MetricItemCategoryUDP:
this.hasUDPMetrics = true
}
}
}
// Add 添加数据
func (this *Manager) Add(obj MetricInterface) {
this.locker.RLock()
for _, task := range this.categoryTasks[obj.MetricCategory()] {
task.Add(obj)
}
this.locker.RUnlock()
}
func (this *Manager) HasHTTPMetrics() bool {
return this.hasHTTPMetrics
}
func (this *Manager) HasTCPMetrics() bool {
return this.hasTCPMetrics
}
func (this *Manager) HasUDPMetrics() bool {
return this.hasUDPMetrics
}

View File

@@ -0,0 +1,63 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package metrics
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"testing"
)
func TestNewManager(t *testing.T) {
var manager = NewManager()
{
manager.Update([]*serverconfigs.MetricItemConfig{})
for _, task := range manager.tasks {
t.Log(task.item.Id)
}
}
{
t.Log("====")
manager.Update([]*serverconfigs.MetricItemConfig{
{
Id: 1,
},
{
Id: 2,
},
{
Id: 3,
},
})
for _, task := range manager.tasks {
t.Log("task:", task.item.Id)
}
}
{
t.Log("====")
manager.Update([]*serverconfigs.MetricItemConfig{
{
Id: 1,
},
{
Id: 2,
},
})
for _, task := range manager.tasks {
t.Log("task:", task.item.Id)
}
}
{
t.Log("====")
manager.Update([]*serverconfigs.MetricItemConfig{
{
Id: 1,
Version: 1,
},
})
for _, task := range manager.tasks {
t.Log("task:", task.item.Id)
}
}
}

View File

@@ -0,0 +1,17 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package metrics
type MetricInterface interface {
// MetricKey 指标对象
MetricKey(key string) string
// MetricValue 指标值
MetricValue(value string) (result int64, ok bool)
// MetricServerId 服务ID
MetricServerId() int64
// MetricCategory 指标分类
MetricCategory() string
}

22
internal/metrics/stat.go Normal file
View File

@@ -0,0 +1,22 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package metrics
import (
"github.com/cespare/xxhash"
"strconv"
)
type Stat struct {
ServerId int64
Keys []string
Hash string
Value int64
Time string
keysData []byte
}
func (this *Stat) Sum(version int32, itemId int64) {
this.Hash = strconv.FormatUint(xxhash.Sum64String(strconv.FormatInt(this.ServerId, 10)+"@"+string(this.keysData)+"@"+this.Time+"@"+strconv.Itoa(int(version))+"@"+strconv.FormatInt(itemId, 10)), 10)
}

492
internal/metrics/task.go Normal file
View File

@@ -0,0 +1,492 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package metrics
import (
"database/sql"
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
"strings"
"sync"
"time"
)
// Task 单个指标任务
// 数据库存储:
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 只上传每个服务中排序最前面的 N 个数据
type Task struct {
item *serverconfigs.MetricItemConfig
isLoaded bool
db *sql.DB
statTableName string
statsChan chan *Stat
isStopped bool
cleanTicker *utils.Ticker
uploadTicker *utils.Ticker
cleanVersion int32
insertStatStmt *sql.Stmt
deleteByVersionStmt *sql.Stmt
deleteByExpiresTimeStmt *sql.Stmt
selectTopStmt *sql.Stmt
sumStmt *sql.Stmt
serverIdMap map[int64]bool // 所有的服务Ids
timeMap map[string]bool // time => bool
serverIdMapLocker sync.Mutex
}
// NewTask 获取新任务
func NewTask(item *serverconfigs.MetricItemConfig) *Task {
return &Task{
item: item,
statsChan: make(chan *Stat, 40960),
serverIdMap: map[int64]bool{},
timeMap: map[string]bool{},
}
}
// Init 初始化
func (this *Task) Init() error {
this.statTableName = "stats"
// 检查目录是否存在
var dir = Tea.Root + "/data"
_, err := os.Stat(dir)
if err != nil {
err = os.MkdirAll(dir, 0777)
if err != nil {
return err
}
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL")
if err != nil {
return err
}
db.SetMaxOpenConns(1)
this.db = db
//创建统计表
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.statTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"keys" varchar(1024),
"value" real DEFAULT 0,
"time" varchar(32),
"serverId" integer DEFAULT 0,
"version" integer DEFAULT 0,
"isUploaded" integer DEFAULT 0
);
CREATE INDEX IF NOT EXISTS "serverId"
ON "` + this.statTableName + `" (
"serverId" ASC,
"version" ASC
);
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
ON "` + this.statTableName + `" (
"hash" ASC
);`)
if err != nil {
return err
}
// insert stat stmt
this.insertStatStmt, err = db.Prepare(`INSERT INTO "stats" ("serverId", "hash", "keys", "value", "time", "version", "isUploaded") VALUES (?, ?, ?, ?, ?, ?, 0) ON CONFLICT("hash") DO UPDATE SET "value"="value"+?, "isUploaded"=0`)
if err != nil {
return err
}
// delete by version
this.deleteByVersionStmt, err = db.Prepare(`DELETE FROM "` + this.statTableName + `" WHERE "version"<?`)
if err != nil {
return err
}
// delete by expires time
this.deleteByExpiresTimeStmt, err = db.Prepare(`DELETE FROM "` + this.statTableName + `" WHERE "time"<?`)
if err != nil {
return err
}
// select topN stmt
this.selectTopStmt, err = db.Prepare(`SELECT "id", "hash", "keys", "value", "isUploaded" FROM "` + this.statTableName + `" WHERE "serverId"=? AND "version"=? AND time=? ORDER BY "value" DESC LIMIT 100`)
if err != nil {
return err
}
// sum stmt
this.sumStmt, err = db.Prepare(`SELECT COUNT(*), IFNULL(SUM(value), 0) FROM "` + this.statTableName + `" WHERE "serverId"=? AND "version"=? AND time=?`)
if err != nil {
return err
}
// 所有的服务IDs
err = this.loadServerIdMap()
if err != nil {
return err
}
this.isLoaded = true
return nil
}
// Start 启动任务
func (this *Task) Start() error {
// 读取数据
go func() {
for stat := range this.statsChan {
if stat == nil {
return
}
err := this.InsertStat(stat)
if err != nil {
remotelogs.Error("METRIC", "insert stat failed: "+err.Error())
}
}
}()
// 清理
this.cleanTicker = utils.NewTicker(24 * time.Hour)
go func() {
for this.cleanTicker.Next() {
err := this.CleanExpired()
if err != nil {
remotelogs.Error("METRIC", "clean expired stats failed: "+err.Error())
}
}
}()
// 上传
this.uploadTicker = utils.NewTicker(this.item.UploadDuration())
go func() {
for this.uploadTicker.Next() {
err := this.Upload(1 * time.Second)
if err != nil {
remotelogs.Error("METRIC", "upload stats failed: "+err.Error())
}
}
}()
return nil
}
// Add 添加数据
func (this *Task) Add(obj MetricInterface) {
if this.isStopped || !this.isLoaded {
return
}
var keys = []string{}
for _, key := range this.item.Keys {
k := obj.MetricKey(key)
keys = append(keys, k)
}
v, ok := obj.MetricValue(this.item.Value)
if !ok {
return
}
var stat = &Stat{
ServerId: obj.MetricServerId(),
Keys: keys,
Value: v,
Time: this.item.CurrentTime(),
}
select {
case this.statsChan <- stat:
default:
// 丢弃
}
}
// Stop 停止任务
func (this *Task) Stop() error {
this.isStopped = true
if this.cleanTicker != nil {
this.cleanTicker.Stop()
}
if this.uploadTicker != nil {
this.uploadTicker.Stop()
}
_ = this.insertStatStmt.Close()
_ = this.deleteByVersionStmt.Close()
_ = this.deleteByExpiresTimeStmt.Close()
_ = this.selectTopStmt.Close()
_ = this.sumStmt.Close()
if this.statsChan != nil {
go func() {
// 延时关闭,防止关闭时写入
time.Sleep(5 * time.Second)
close(this.statsChan)
}()
}
if this.db != nil {
_ = this.db.Close()
}
return nil
}
// InsertStat 写入数据
func (this *Task) InsertStat(stat *Stat) error {
if this.isStopped {
return nil
}
if stat == nil {
return nil
}
this.serverIdMapLocker.Lock()
this.serverIdMap[stat.ServerId] = true
this.timeMap[stat.Time] = true
this.serverIdMapLocker.Unlock()
keyData, err := json.Marshal(stat.Keys)
if err != nil {
return err
}
stat.keysData = keyData
stat.Sum(this.item.Version, this.item.Id)
_, err = this.insertStatStmt.Exec(stat.ServerId, stat.Hash, stat.keysData, stat.Value, stat.Time, this.item.Version, stat.Value)
if err != nil {
return err
}
return nil
}
// CleanExpired 清理数据
func (this *Task) CleanExpired() error {
if this.isStopped {
return nil
}
// 清除低版本数据
if this.cleanVersion < this.item.Version {
_, err := this.deleteByVersionStmt.Exec(this.item.Version)
if err != nil {
return err
}
this.cleanVersion = this.item.Version
}
// 清除过期的数据
_, err := this.deleteByExpiresTimeStmt.Exec(this.item.LocalExpiresTime())
if err != nil {
return err
}
return nil
}
// Upload 上传数据
func (this *Task) Upload(pauseDuration time.Duration) error {
if this.isStopped {
return nil
}
this.serverIdMapLocker.Lock()
// 服务IDs
var serverIds []int64
for serverId := range this.serverIdMap {
serverIds = append(serverIds, serverId)
}
this.serverIdMap = map[int64]bool{} // 清空数据
// 时间
var times = []string{}
for t := range this.timeMap {
times = append(times, t)
}
this.timeMap = map[string]bool{} // 清空数据
this.serverIdMapLocker.Unlock()
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
for _, serverId := range serverIds {
for _, currentTime := range times {
idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
rows, err := this.selectTopStmt.Query(serverId, this.item.Version, currentTime)
if err != nil {
return nil, err
}
var isClosed bool
defer func() {
if isClosed {
return
}
_ = rows.Close()
}()
var pbStats []*pb.UploadingMetricStat
for rows.Next() {
var pbStat = &pb.UploadingMetricStat{
}
// "id", "hash", "keys", "value", "isUploaded"
var isUploaded int
var keysData []byte
err = rows.Scan(&pbStat.Id, &pbStat.Hash, &keysData, &pbStat.Value, &isUploaded)
if err != nil {
return nil, err
}
if isUploaded == 1 {
continue
}
if len(keysData) > 0 {
err = json.Unmarshal(keysData, &pbStat.Keys)
if err != nil {
return nil, err
}
}
pbStats = append(pbStats, pbStat)
ids = append(ids, strconv.FormatInt(pbStat.Id, 10))
}
// 提前关闭
_ = rows.Close()
isClosed = true
// 上传
if len(pbStats) > 0 {
// 计算总和
count, total, err := this.sum(serverId, currentTime)
if err != nil {
return nil, err
}
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
MetricStats: pbStats,
Time: currentTime,
ServerId: serverId,
ItemId: this.item.Id,
Version: this.item.Version,
Count: count,
Total: float32(total),
})
if err != nil {
return nil, err
}
}
return
}(serverId, currentTime)
if err != nil {
return err
}
if len(idStrings) > 0 {
// 设置为已上传
_, err = this.db.Exec(`UPDATE "` + this.statTableName + `" SET isUploaded=1 WHERE id IN (` + strings.Join(idStrings, ",") + `)`)
if err != nil {
return err
}
}
}
// 休息一下,防止短时间内上传数据过多
if pauseDuration > 0 {
time.Sleep(pauseDuration)
}
}
return nil
}
// 加载服务ID
func (this *Task) loadServerIdMap() error {
{
rows, err := this.db.Query(`SELECT DISTINCT "serverId" FROM `+this.statTableName+" WHERE version=?", this.item.Version)
if err != nil {
return err
}
defer func() {
_ = rows.Close()
}()
var serverId int64
for rows.Next() {
err = rows.Scan(&serverId)
if err != nil {
return err
}
this.serverIdMapLocker.Lock()
this.serverIdMap[serverId] = true
this.serverIdMapLocker.Unlock()
}
}
{
rows, err := this.db.Query(`SELECT DISTINCT "time" FROM `+this.statTableName+" WHERE version=?", this.item.Version)
if err != nil {
return err
}
defer func() {
_ = rows.Close()
}()
var timeString string
for rows.Next() {
err = rows.Scan(&timeString)
if err != nil {
return err
}
this.serverIdMapLocker.Lock()
this.timeMap[timeString] = true
this.serverIdMapLocker.Unlock()
}
}
return nil
}
// 计算数量和综合
func (this *Task) sum(serverId int64, time string) (count int64, total float64, err error) {
rows, err := this.sumStmt.Query(serverId, this.item.Version, time)
if err != nil {
return 0, 0, err
}
defer func() {
_ = rows.Close()
}()
if rows.Next() {
err = rows.Scan(&count, &total)
if err != nil {
return 0, 0, err
}
}
return
}

View File

@@ -0,0 +1,210 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package metrics_test
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"testing"
"time"
)
type testObj struct {
ip string
}
func (this *testObj) MetricKey(key string) string {
return this.ip
}
func (this *testObj) MetricValue(value string) (int64, bool) {
return 1, true
}
func (this *testObj) MetricServerId() int64 {
return int64(rands.Int(1, 100))
}
func (this *testObj) MetricCategory() string {
return "http"
}
func TestTask_Init(t *testing.T) {
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
Id: 1,
IsOn: false,
Category: "",
Period: 0,
PeriodUnit: "",
Keys: nil,
Value: "",
})
err := task.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = task.Stop()
}()
t.Log("ok")
}
func TestTask_Add(t *testing.T) {
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
Id: 1,
IsOn: false,
Category: "",
Period: 1,
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
Keys: []string{"${remoteAddr}"},
Value: "${countRequest}",
})
err := task.Init()
if err != nil {
t.Fatal(err)
}
err = task.Start()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = task.Stop()
}()
task.Add(&testObj{ip: "127.0.0.2"})
time.Sleep(1 * time.Second) // waiting for inserting
}
func TestTask_Add_Many(t *testing.T) {
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
Id: 1,
IsOn: false,
Category: "",
Period: 1,
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
Keys: []string{"${remoteAddr}"},
Value: "${countRequest}",
Version: 1,
})
err := task.Init()
if err != nil {
t.Fatal(err)
}
err = task.Start()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = task.Stop()
}()
for i := 0; i < 4_000_000; i++ {
task.Add(&testObj{
ip: fmt.Sprintf("%d.%d.%d.%d", rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255)),
})
if i%10000 == 0 {
time.Sleep(1 * time.Second)
}
}
}
func TestTask_InsertStat(t *testing.T) {
var item = &serverconfigs.MetricItemConfig{
Id: 1,
IsOn: false,
Category: "",
Period: 1,
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
Keys: []string{"${remoteAddr}"},
Value: "${countRequest}",
Version: 1,
}
var task = metrics.NewTask(item)
err := task.Init()
if err != nil {
t.Fatal(err)
}
err = task.Start()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = task.Stop()
}()
err = task.InsertStat(&metrics.Stat{
ServerId: 1,
Keys: []string{"127.0.0.1"},
Hash: "",
Value: 1,
Time: item.CurrentTime(),
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestTask_CleanExpired(t *testing.T) {
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
Id: 1,
IsOn: false,
Category: "",
Period: 1,
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
Keys: []string{"${remoteAddr}"},
Value: "${countRequest}",
Version: 1,
})
err := task.Init()
if err != nil {
t.Fatal(err)
}
err = task.Start()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = task.Stop()
}()
err = task.CleanExpired()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestTask_Upload(t *testing.T) {
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
Id: 1,
IsOn: false,
Category: "",
Period: 1,
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
Keys: []string{"${remoteAddr}"},
Value: "${countRequest}",
Version: 1,
})
err := task.Init()
if err != nil {
t.Fatal(err)
}
err = task.Start()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = task.Stop()
}()
err = task.Upload(0)
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -72,7 +72,8 @@ func (this *ValueQueue) Loop() error {
CreatedAt: value.CreatedAt,
})
if err != nil {
return err
remotelogs.Error("MONITOR", err.Error())
continue
}
}
return nil

View File

@@ -1,6 +1,7 @@
package nodes
import (
"context"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
@@ -13,6 +14,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/logs"
"io"
"net/http"
"os/exec"
@@ -55,10 +57,16 @@ func (this *APIStream) loop() error {
return errors.Wrap(err)
}
isQuiting := false
ctx, cancelFunc := context.WithCancel(rpcClient.Context())
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
events.On(events.EventQuit, func() {
isQuiting = true
remotelogs.Println("API_STREAM", "quiting")
if nodeStream != nil {
cancelFunc()
}
})
nodeStream, err := rpcClient.NodeRPC().NodeStream(rpcClient.Context())
if err != nil {
if isQuiting {
return nil
@@ -69,12 +77,14 @@ func (this *APIStream) loop() error {
for {
if isQuiting {
logs.Println("API_STREAM", "quit")
break
}
message, err := nodeStream.Recv()
if err != nil {
if isQuiting {
remotelogs.Println("API_STREAM", "quit")
return nil
}
return errors.Wrap(err)

View File

@@ -77,7 +77,7 @@ Loop:
return err
}
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{AccessLogs: accessLogs})
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
if err != nil {
return err
}

View File

@@ -1,14 +1,17 @@
package nodes
import (
"context"
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/types"
"io"
"net"
"net/http"
"net/url"
@@ -28,6 +31,9 @@ var bytePool1k = utils.NewBytePool(20480, 1024)
var bytePool32k = utils.NewBytePool(20480, 32*1024)
var bytePool128k = utils.NewBytePool(20480, 128*1024)
// errors
var errWritingToClient = errors.New("writing to client error")
// HTTPRequest HTTP请求
type HTTPRequest struct {
// 外部参数
@@ -41,6 +47,7 @@ type HTTPRequest struct {
IsHTTPS bool
// 内部参数
isSubRequest bool
writer *HTTPWriter
web *serverconfigs.HTTPWebConfig // Web配置重要提示由于引用了别的共享的配置所以操作中只能读取不要修改
reverseProxyRef *serverconfigs.ReverseProxyRef // 反向代理引用
@@ -59,12 +66,17 @@ type HTTPRequest struct {
rewriteIsExternalURL bool // 重写目标是否为外部URL
cacheRef *serverconfigs.HTTPCacheRef // 缓存设置
cacheKey string // 缓存使用的Key
isCached bool // 是否已经被缓存
isAttack bool // 是否是攻击请求
bodyData []byte // 读取的Body内容
// WAF相关
firewallPolicyId int64
firewallRuleGroupId int64
firewallRuleSetId int64
firewallRuleId int64
firewallActions []string
tags []string
logAttrs map[string]string
@@ -123,7 +135,12 @@ func (this *HTTPRequest) Do() {
}
// 访问控制
// TODO 需要实现
if !this.isSubRequest && this.web.Auth != nil && this.web.Auth.IsOn {
if this.doAuth() {
this.doEnd()
return
}
}
// 自动跳转到HTTPS
if this.IsHTTP && this.web.RedirectToHttps != nil && this.web.RedirectToHttps.IsOn {
@@ -229,7 +246,20 @@ func (this *HTTPRequest) doEnd() {
// 流量统计
// TODO 增加是否开启开关
if this.Server != nil {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.writer.sentBodyBytes)
if this.isCached {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, this.writer.sentBodyBytes, 1, 1, 0, 0)
} else {
if this.isAttack {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 1, this.writer.sentBodyBytes)
} else {
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 0, 0)
}
}
}
// 指标
if metrics.SharedManager.HasHTTPMetrics() {
this.doMetricsResponse()
}
}
@@ -342,6 +372,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
this.web.FastcgiList = web.FastcgiList
}
// auth
if web.Auth != nil && (web.Auth.IsPrior || isTop) {
this.web.Auth = web.Auth
}
// 重写规则
if len(web.RewriteRefs) > 0 {
for index, ref := range web.RewriteRefs {
@@ -479,9 +514,15 @@ func (this *HTTPRequest) Format(source string) string {
return this.requestRemoteUser()
case "requestURI", "requestUri":
return this.rawURI
case "requestURL":
var scheme = "http"
if this.IsHTTPS {
scheme = "https"
}
return scheme + "://" + this.Host + this.rawURI
case "requestPath":
return this.requestPath()
case "requestPathExtension": // TODO 需要添加到文档中
case "requestPathExtension":
return filepath.Ext(this.requestPath())
case "requestLength":
return strconv.FormatInt(this.requestLength(), 10)
@@ -576,7 +617,6 @@ func (this *HTTPRequest) Format(source string) string {
}
// response.
// TODO 需要在文档中添加说明
if prefix == "response" {
switch suffix {
case "contentType":
@@ -910,6 +950,11 @@ func (this *HTTPRequest) requestServerPort() int {
return 0
}
// 获取完整的URL
func (this *HTTPRequest) requestFullURL() string {
return this.requestScheme() + "://" + this.Host + this.uri
}
// 设置代理相关头部信息
// 参考https://tools.ietf.org/html/rfc7239
func (this *HTTPRequest) setForwardHeaders(header http.Header) {
@@ -1133,3 +1178,38 @@ func (this *HTTPRequest) bytePool(contentLength int64) *utils.BytePool {
}
return bytePool128k
}
// 检查是否可以忽略错误
func (this *HTTPRequest) canIgnore(err error) bool {
if err == nil {
return true
}
// 已读到头
if err == io.EOF || err == io.ErrUnexpectedEOF {
return true
}
// 网络错误
_, ok := err.(*net.OpError)
if ok {
return true
}
// 客户端主动取消
if err == errWritingToClient || err == context.Canceled || err == io.ErrShortWrite || strings.Contains(err.Error(), "write: connection timed out") || strings.Contains(err.Error(), "write: broken pipe") {
return true
}
// HTTP/2流错误
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
return true
}
// HTTP内部错误
if strings.HasPrefix(err.Error(), "http:") || strings.HasPrefix(err.Error(), "http2:") {
return true
}
return false
}

View File

@@ -0,0 +1,61 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"io/ioutil"
"net/http"
)
// 执行认证
func (this *HTTPRequest) doAuth() (shouldStop bool) {
if this.web.Auth == nil || !this.web.Auth.IsOn {
return
}
for _, ref := range this.web.Auth.PolicyRefs {
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
continue
}
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
subReq.TLS = this.RawReq.TLS
subReq.RemoteAddr = this.RawReq.RemoteAddr
subReq.Host = this.RawReq.Host
subReq.Proto = this.RawReq.Proto
subReq.ProtoMinor = this.RawReq.ProtoMinor
subReq.ProtoMajor = this.RawReq.ProtoMajor
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
subReq.Header.Set("Referer", this.requestFullURL())
var writer = NewEmptyResponseWriter(this.writer)
this.doSubRequest(writer, subReq)
return writer.StatusCode(), nil
}, this.Format)
if err != nil {
this.write502(err)
return
}
if b {
return
} else {
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.Host
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
this.writer.WriteHeader(http.StatusUnauthorized)
return true
}
}
return
}

View File

@@ -5,14 +5,19 @@ import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/logs"
"net/http"
"strconv"
"time"
)
// 读取缓存
func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
if this.web.Cache == nil || !this.web.Cache.IsOn || len(this.web.Cache.CacheRefs) == 0 {
cachePolicy := sharedNodeConfig.HTTPCachePolicy
if cachePolicy == nil || !cachePolicy.IsOn {
return
}
if this.web.Cache == nil || !this.web.Cache.IsOn || (len(cachePolicy.CacheRefs) == 0 && len(this.web.Cache.CacheRefs) == 0) {
return
}
var addStatusHeader = this.web.Cache.AddStatusHeader
@@ -25,12 +30,8 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
}()
}
cachePolicy := sharedNodeConfig.HTTPCachePolicy
if cachePolicy == nil || !cachePolicy.IsOn {
return
}
// 检查条件
// 检查服务独立的缓存条件
refType := ""
for _, cacheRef := range this.web.Cache.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
@@ -38,12 +39,35 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if cacheRef.IsReverse {
return
}
this.cacheRef = cacheRef
refType = "server"
break
}
}
if this.cacheRef == nil {
return
// 检查策略默认的缓存条件
for _, cacheRef := range cachePolicy.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if cacheRef.IsReverse {
return
}
this.cacheRef = cacheRef
refType = "policy"
break
}
}
if this.cacheRef == nil {
return
}
}
// 相关变量
@@ -89,7 +113,9 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
return
}
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
}
return
}
defer func() {
@@ -121,13 +147,58 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
return true, nil
})
if err != nil {
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
}
return
}
if addStatusHeader {
this.writer.Header().Set("X-Cache", "HIT")
this.writer.Header().Set("X-Cache", "HIT, "+refType+", "+reader.TypeName())
}
// ETag
var respHeader = this.writer.Header()
var eTag = respHeader.Get("ETag")
var lastModifiedAt = reader.LastModified()
if len(eTag) == 0 {
if lastModifiedAt > 0 {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
respHeader["ETag"] = []string{eTag}
}
}
// 支持 Last-Modified
var modifiedTime = respHeader.Get("Last-Modified")
if len(modifiedTime) == 0 {
if lastModifiedAt > 0 {
modifiedTime = time.Unix(lastModifiedAt, 0).Format("Mon, 02 Jan 2006 15:04:05 GMT")
if len(respHeader.Get("Last-Modified")) == 0 {
respHeader.Set("Last-Modified", modifiedTime)
}
}
}
// 支持 If-None-Match
if len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.writer.WriteHeader(http.StatusNotModified)
this.isCached = true
this.cacheRef = nil
return true
}
// 支持 If-Modified-Since
if len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.writer.WriteHeader(http.StatusNotModified)
this.isCached = true
this.cacheRef = nil
return true
}
this.processResponseHeaders(reader.Status())
// 输出Body
@@ -201,7 +272,7 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
err = reader.ReadBodyRange(buf, rangeSet[0][0], rangeSet[0][1], func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, err
return false, errWritingToClient
}
return true, nil
})
@@ -211,7 +282,9 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
}
return
}
} else if len(rangeSet) > 1 {
@@ -229,37 +302,42 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
_, err = this.writer.WriteString("\r\n--" + boundary + "\r\n")
}
if err != nil {
logs.Error(err)
// 不提示写入客户端错误
return true
}
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(reader.BodySize(), 10) + "\r\n")
if err != nil {
logs.Error(err)
// 不提示写入客户端错误
return true
}
if len(contentType) > 0 {
_, err = this.writer.WriteString("Content-Type: " + contentType + "\r\n\r\n")
if err != nil {
logs.Error(err)
// 不提示写入客户端错误
return true
}
}
err := reader.ReadBodyRange(buf, set[0], set[1], func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
return true, err
if err != nil {
return false, errWritingToClient
}
return true, nil
})
if err != nil {
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
}
return true
}
}
_, err = this.writer.WriteString("\r\n--" + boundary + "--\r\n")
if err != nil {
logs.Error(err)
// 不提示写入客户端错误
return true
}
} else { // 没有Range
@@ -268,17 +346,20 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, err
return false, errWritingToClient
}
return true, nil
})
if err != nil {
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
}
return
}
}
}
this.cacheRef = nil // 终止读取不再往下传递
this.isCached = true
this.cacheRef = nil
return true
}

View File

@@ -12,7 +12,7 @@ import (
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"github.com/iwind/gofcgi/pkg"
"github.com/iwind/gofcgi/pkg/fcgi"
"io"
"net"
"net/url"
@@ -78,7 +78,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
poolSize = 32
}
client, err := pkg.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
client, err := fcgi.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
if err != nil {
this.write500(err)
return
@@ -151,7 +151,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
params["HTTP_HOST"] = this.Host
}
fcgiReq := pkg.NewRequest()
fcgiReq := fcgi.NewRequest()
fcgiReq.SetTimeout(fastcgi.ReadTimeoutDuration())
fcgiReq.SetParams(params)
fcgiReq.SetBody(this.RawReq.Body, uint32(this.requestLength()))
@@ -189,7 +189,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
this.processResponseHeaders(resp.StatusCode)
// 准备
this.writer.Prepare(resp.ContentLength)
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
// 设置响应代码
this.writer.WriteHeader(resp.StatusCode)
@@ -200,14 +200,20 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
pool.Put(buf)
err1 := resp.Body.Close()
if err1 != nil {
remotelogs.Error("REQUEST_REVERSE_PROXY", err1.Error())
closeErr := resp.Body.Close()
if closeErr != nil {
remotelogs.Warn("HTTP_REQUEST_FASTCGI", closeErr.Error())
}
if err != nil && err != io.EOF {
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
remotelogs.Warn("HTTP_REQUEST_FASTCGI", err.Error())
this.addError(err)
}
// 是否成功结束
if err == nil && closeErr == nil {
this.writer.SetOk()
}
return
}

View File

@@ -2,6 +2,7 @@ package nodes
import (
"net/http"
"strconv"
"strings"
)
@@ -12,7 +13,10 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
if !u.IsOn {
continue
}
if u.MatchPrefix {
if !u.MatchRequest(this.Format) {
continue
}
if u.MatchPrefix { // 匹配前缀
if strings.HasPrefix(fullURL, u.BeforeURL) {
afterURL := u.AfterURL
if u.KeepRequestURI {
@@ -25,7 +29,39 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
}
return true
}
} else {
} else if u.MatchRegexp { // 正则匹配
reg := u.BeforeURLRegexp()
if reg == nil {
continue
}
matches := reg.FindStringSubmatch(fullURL)
if len(matches) == 0 {
continue
}
afterURL := u.AfterURL
for i, match := range matches {
afterURL = strings.ReplaceAll(afterURL, "${"+strconv.Itoa(i)+"}", match)
}
subNames := reg.SubexpNames()
if len(subNames) > 0 {
for _, subName := range subNames {
if len(subName) > 0 {
index := reg.SubexpIndex(subName)
if index > -1 {
afterURL = strings.ReplaceAll(afterURL, "${"+subName+"}", matches[index])
}
}
}
}
if u.Status <= 0 {
http.Redirect(this.RawWriter, this.RawReq, afterURL, http.StatusTemporaryRedirect)
} else {
http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
}
return true
} else { // 精准匹配
if fullURL == u.RealBeforeURL() {
if u.Status <= 0 {
http.Redirect(this.RawWriter, this.RawReq, u.AfterURL, http.StatusTemporaryRedirect)

View File

@@ -128,6 +128,8 @@ func (this *HTTPRequest) log() {
FirewallRuleGroupId: this.firewallRuleGroupId,
FirewallRuleSetId: this.firewallRuleSetId,
FirewallRuleId: this.firewallRuleId,
FirewallActions: this.firewallActions,
Tags: this.tags,
Attrs: this.logAttrs,
}

View File

@@ -0,0 +1,58 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
)
// 指标统计 - 响应
// 只需要在结束时调用指标进行统计
func (this *HTTPRequest) doMetricsResponse() {
metrics.SharedManager.Add(this)
}
func (this *HTTPRequest) MetricKey(key string) string {
return this.Format(key)
}
func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
// TODO 需要忽略健康检查的请求,但是同时也要防止攻击者模拟健康检查
switch value {
case "${countRequest}":
return 1, true
case "${countTrafficOut}":
// 这里不包括Header长度
return this.writer.SentBodyBytes(), true
case "${countTrafficIn}":
var hl int64 = 0 // header length
for k, values := range this.RawReq.Header {
for _, v := range values {
hl += int64(len(k) + len(v) + 2 /** k: v **/)
}
}
return this.RawReq.ContentLength + hl, true
case "${countConnection}":
metricNewConnMapLocker.Lock()
_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
if ok {
delete(metricNewConnMap, this.RawReq.RemoteAddr)
}
metricNewConnMapLocker.Unlock()
if ok {
return 1, true
} else {
return 0, false
}
}
return 0, false
}
func (this *HTTPRequest) MetricServerId() int64 {
return this.Server.Id
}
func (this *HTTPRequest) MetricCategory() string {
return serverconfigs.MetricItemCategoryHTTP
}

View File

@@ -1,6 +1,7 @@
package nodes
import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"io"
@@ -50,7 +51,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
_, err = io.CopyBuffer(this.writer, fp, buf)
bytePool1k.Put(buf)
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_PAGE", "write to client failed: "+err.Error())
}
} else {
this.writer.SetOk()
}
err = fp.Close()
if err != nil {

View File

@@ -19,11 +19,10 @@ func (this *HTTPRequest) doRedirectToHTTPS(redirectToHTTPSConfig *serverconfigs.
} else if redirectToHTTPSConfig.Port > 0 {
lastIndex := strings.LastIndex(host, ":")
if lastIndex > 0 {
if redirectToHTTPSConfig.Port != 443 {
host = host[:lastIndex] + ":" + strconv.Itoa(redirectToHTTPSConfig.Port)
} else {
host = host[:lastIndex]
}
host = host[:lastIndex]
}
if redirectToHTTPSConfig.Port != 443 {
host = host + ":" + strconv.Itoa(redirectToHTTPSConfig.Port)
}
} else {
lastIndex := strings.LastIndex(host, ":")

View File

@@ -35,7 +35,7 @@ func (this *HTTPRequest) doReverseProxy() {
origin := this.reverseProxy.NextOrigin(requestCall)
if origin == nil {
err := errors.New(this.requestPath() + ": no available backends for reverse proxy")
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write502(err)
return
}
@@ -55,7 +55,7 @@ func (this *HTTPRequest) doReverseProxy() {
// 处理Scheme
if origin.Addr == nil {
err := errors.New(this.requestPath() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write502(err)
return
}
@@ -142,7 +142,7 @@ func (this *HTTPRequest) doReverseProxy() {
// 获取请求客户端
client, err := SharedHTTPClientPool.Client(this.RawReq, origin, originAddr)
if err != nil {
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write502(err)
return
}
@@ -162,7 +162,7 @@ func (this *HTTPRequest) doReverseProxy() {
// TODO 如果超过最大失败次数,则下线
this.write502(err)
remotelogs.Println("REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
} else {
// 是否为客户端方面的错误
isClientError := false
@@ -189,7 +189,7 @@ func (this *HTTPRequest) doReverseProxy() {
if this.doWAFResponse(resp) {
err = resp.Body.Close()
if err != nil {
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", err.Error())
}
return
}
@@ -201,7 +201,7 @@ func (this *HTTPRequest) doReverseProxy() {
if len(this.web.Pages) > 0 && this.doPage(resp.StatusCode) {
err = resp.Body.Close()
if err != nil {
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", err.Error())
}
return
}
@@ -226,7 +226,7 @@ func (this *HTTPRequest) doReverseProxy() {
shouldAutoFlush := this.reverseProxy.AutoFlush || this.RawReq.Header.Get("Accept") == "text/event-stream"
// 准备
this.writer.Prepare(resp.ContentLength)
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
// 设置响应代码
this.writer.WriteHeader(resp.StatusCode)
@@ -254,13 +254,22 @@ func (this *HTTPRequest) doReverseProxy() {
}
pool.Put(buf)
err1 := resp.Body.Close()
if err1 != nil {
remotelogs.Error("REQUEST_REVERSE_PROXY", err1.Error())
closeErr := resp.Body.Close()
if closeErr != nil {
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", closeErr.Error())
}
}
if err != nil && err != io.EOF {
remotelogs.Error("REQUEST_REVERSE_PROXY", err.Error())
this.addError(err)
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.addError(err)
}
}
// 是否成功结束
if err == nil && closeErr == nil {
this.writer.SetOk()
}
}

View File

@@ -296,7 +296,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.cacheRef = nil // 不支持缓存
}
this.writer.Prepare(fileSize)
this.writer.Prepare(fileSize, http.StatusOK)
pool := this.bytePool(fileSize)
buf := pool.Get()
@@ -382,6 +382,9 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
}
// 设置成功
this.writer.SetOk()
return true
}

View File

@@ -1,6 +1,7 @@
package nodes
import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"io"
@@ -65,8 +66,16 @@ func (this *HTTPRequest) doShutdown() {
buf := bytePool1k.Get()
_, err = io.CopyBuffer(this.writer, fp, buf)
bytePool1k.Put(buf)
if err != nil {
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_SHUTDOWN", "write to client failed: "+err.Error())
}
} else {
this.writer.SetOk()
}
err = fp.Close()
if err != nil {
logs.Error(err)
remotelogs.Warn("HTTP_REQUEST_SHUTDOWN", "close file failed: "+err.Error())
}
}

View File

@@ -7,6 +7,8 @@ func (this *HTTPRequest) doStat() {
if this.Server == nil {
return
}
// 内置的统计
stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.Server.Id, this.requestRemoteAddr())
stats.SharedHTTPRequestStatManager.AddUserAgent(this.Server.Id, this.requestHeader("User-Agent"))
}

View File

@@ -0,0 +1,22 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import "net/http"
// 执行子请求
func (this *HTTPRequest) doSubRequest(writer http.ResponseWriter, rawReq *http.Request) {
// 包装新请求对象
req := &HTTPRequest{
RawReq: rawReq,
RawWriter: writer,
Server: this.Server,
Host: this.Host,
ServerName: this.ServerName,
ServerAddr: this.ServerAddr,
IsHTTP: this.IsHTTP,
IsHTTPS: this.IsHTTPS,
}
req.isSubRequest = true
req.Do()
}

View File

@@ -2,6 +2,7 @@ package nodes
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/logs"
"io"
@@ -50,7 +51,11 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
}
this.writer.AddHeaders(resp.Header)
this.writer.Prepare(resp.ContentLength)
if statusCode <= 0 {
this.writer.Prepare(resp.ContentLength, resp.StatusCode)
} else {
this.writer.Prepare(resp.ContentLength, statusCode)
}
// 设置响应代码
if statusCode <= 0 {
@@ -64,4 +69,12 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
buf := pool.Get()
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
pool.Put(buf)
if err != nil {
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_URL", "write to client failed: "+err.Error())
}
} else {
this.writer.SetOk()
}
}

View File

@@ -1,6 +1,7 @@
package nodes
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -8,6 +9,8 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"io"
"io/ioutil"
"net/http"
)
@@ -47,48 +50,55 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// 检查IP白名单
remoteAddrs := this.requestRemoteAddrs()
inbound := firewallPolicy.Inbound
if inbound.AllowListRef != nil && inbound.AllowListRef.IsOn && inbound.AllowListRef.ListId > 0 {
list := iplibrary.SharedIPListManager.FindList(inbound.AllowListRef.ListId)
if list != nil {
found, _ := list.ContainsIPStrings(remoteAddrs)
if found {
breakChecking = true
return
if inbound == nil {
return
}
for _, ref := range inbound.AllAllowListRefs() {
if ref.IsOn && ref.ListId > 0 {
list := iplibrary.SharedIPListManager.FindList(ref.ListId)
if list != nil {
found, _ := list.ContainsIPStrings(remoteAddrs)
if found {
breakChecking = true
return
}
}
}
}
// 检查IP黑名单
if inbound.DenyListRef != nil && inbound.DenyListRef.IsOn && inbound.DenyListRef.ListId > 0 {
list := iplibrary.SharedIPListManager.FindList(inbound.DenyListRef.ListId)
if list != nil {
found, item := list.ContainsIPStrings(remoteAddrs)
if found {
// 触发事件
if item != nil && len(item.EventLevel) > 0 {
actions := iplibrary.SharedActionManager.FindEventActions(item.EventLevel)
for _, action := range actions {
goNext, err := action.DoHTTP(this.RawReq, this.RawWriter)
if err != nil {
remotelogs.Error("REQUEST", "do action '"+err.Error()+"' failed: "+err.Error())
return true, false
}
if !goNext {
this.disableLog = true
return true, false
for _, ref := range inbound.AllDenyListRefs() {
if ref.IsOn && ref.ListId > 0 {
list := iplibrary.SharedIPListManager.FindList(ref.ListId)
if list != nil {
found, item := list.ContainsIPStrings(remoteAddrs)
if found {
// 触发事件
if item != nil && len(item.EventLevel) > 0 {
actions := iplibrary.SharedActionManager.FindEventActions(item.EventLevel)
for _, action := range actions {
goNext, err := action.DoHTTP(this.RawReq, this.RawWriter)
if err != nil {
remotelogs.Error("HTTP_REQUEST_WAF", "do action '"+err.Error()+"' failed: "+err.Error())
return true, false
}
if !goNext {
this.disableLog = true
return true, false
}
}
}
// TODO 需要记录日志信息
this.writer.WriteHeader(http.StatusForbidden)
this.writer.Close()
// 停止日志
this.disableLog = true
return true, false
}
// TODO 需要记录日志信息
this.writer.WriteHeader(http.StatusForbidden)
this.writer.Close()
// 停止日志
this.disableLog = true
return true, false
}
}
}
@@ -101,7 +111,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
for _, remoteAddr := range remoteAddrs {
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
if err != nil {
remotelogs.Error("REQUEST", "iplibrary lookup failed: "+err.Error())
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
} else if result != nil {
// 检查国家级别封禁
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
@@ -145,23 +155,36 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
if w == nil {
return
}
goNext, ruleGroup, ruleSet, err := w.MatchRequest(this.RawReq, this.writer)
w.OnAction(func(action waf.ActionInterface) (goNext bool) {
switch action.Code() {
case waf.ActionTag:
this.tags = action.(*waf.TagAction).Tags
}
return true
})
goNext, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer)
if err != nil {
remotelogs.Error("REQUEST", this.rawURI+": "+err.Error())
remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
return
}
if ruleSet != nil {
if ruleSet.Action != waf.ActionAllow {
if ruleSet.HasSpecialActions() {
this.firewallPolicyId = firewallPolicy.Id
this.firewallRuleGroupId = types.Int64(ruleGroup.Id)
this.firewallRuleSetId = types.Int64(ruleSet.Id)
if ruleSet.HasAttackActions() {
this.isAttack = true
}
// 添加统计
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Action)
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Actions)
}
this.logAttrs["waf.action"] = ruleSet.Action
this.firewallActions = ruleSet.ActionCodes()
}
return !goNext, false
@@ -169,7 +192,25 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// call response waf
func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
firewallPolicy := sharedNodeConfig.HTTPFirewallPolicy
// 当前服务的独立设置
if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp)
if blocked {
return true
}
}
// 公用的防火墙设置
if sharedNodeConfig.HTTPFirewallPolicy != nil {
blocked := this.checkWAFResponse(sharedNodeConfig.HTTPFirewallPolicy, resp)
if blocked {
return true
}
}
return
}
func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response) (blocked bool) {
if firewallPolicy == nil || !firewallPolicy.IsOn || !firewallPolicy.Outbound.IsOn {
return
}
@@ -179,24 +220,79 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
return
}
goNext, ruleGroup, ruleSet, err := w.MatchResponse(this.RawReq, resp, this.writer)
w.OnAction(func(action waf.ActionInterface) (goNext bool) {
switch action.Code() {
case waf.ActionTag:
this.tags = action.(*waf.TagAction).Tags
}
return true
})
goNext, ruleGroup, ruleSet, err := w.MatchResponse(this, resp, this.writer)
if err != nil {
remotelogs.Error("REQUEST", this.rawURI+": "+err.Error())
remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
return
}
if ruleSet != nil {
if ruleSet.Action != waf.ActionAllow {
if ruleSet.HasSpecialActions() {
this.firewallPolicyId = firewallPolicy.Id
this.firewallRuleGroupId = types.Int64(ruleGroup.Id)
this.firewallRuleSetId = types.Int64(ruleSet.Id)
if ruleSet.HasAttackActions() {
this.isAttack = true
}
// 添加统计
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Action)
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Actions)
}
this.logAttrs["waf.action"] = ruleSet.Action
this.firewallActions = ruleSet.ActionCodes()
}
return !goNext
}
// WAFRaw 原始请求
func (this *HTTPRequest) WAFRaw() *http.Request {
return this.RawReq
}
// WAFRemoteIP 客户端IP
func (this *HTTPRequest) WAFRemoteIP() string {
return this.requestRemoteAddr()
}
// WAFGetCacheBody 获取缓存中的Body
func (this *HTTPRequest) WAFGetCacheBody() []byte {
return this.bodyData
}
// WAFSetCacheBody 设置Body
func (this *HTTPRequest) WAFSetCacheBody(body []byte) {
this.bodyData = body
}
// WAFReadBody 读取Body
func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
if this.RawReq.ContentLength > 0 {
data, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, max))
}
return
}
// WAFRestoreBody 恢复Body
func (this *HTTPRequest) WAFRestoreBody(data []byte) {
if len(data) > 0 {
rawReader := bytes.NewBuffer(data)
buf := make([]byte, 1024)
_, _ = io.CopyBuffer(rawReader, this.RawReq.Body, buf)
this.RawReq.Body = ioutil.NopCloser(rawReader)
}
}
// WAFServerId 服务ID
func (this *HTTPRequest) WAFServerId() int64 {
return this.Server.Id
}

View File

@@ -9,12 +9,13 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"net"
"net/http"
"strings"
)
// 响应Writer
// HTTPWriter 响应Writer
type HTTPWriter struct {
req *HTTPRequest
writer http.ResponseWriter
@@ -32,9 +33,11 @@ type HTTPWriter struct {
cacheWriter caches.Writer // 缓存写入
cacheStorage caches.StorageInterface
isOk bool // 是否完全成功
}
// 包装对象
// NewHTTPWriter 包装对象
func NewHTTPWriter(req *HTTPRequest, httpResponseWriter http.ResponseWriter) *HTTPWriter {
return &HTTPWriter{
req: req,
@@ -42,7 +45,7 @@ func NewHTTPWriter(req *HTTPRequest, httpResponseWriter http.ResponseWriter) *HT
}
}
// 重置
// Reset 重置
func (this *HTTPWriter) Reset(httpResponseWriter http.ResponseWriter) {
this.writer = httpResponseWriter
@@ -58,23 +61,25 @@ func (this *HTTPWriter) Reset(httpResponseWriter http.ResponseWriter) {
this.gzipBodyWriter = nil
}
// 设置Gzip
// Gzip 设置Gzip
func (this *HTTPWriter) Gzip(config *serverconfigs.HTTPGzipConfig) {
this.gzipConfig = config
}
// 准备输出
func (this *HTTPWriter) Prepare(size int64) {
// Prepare 准备输出
func (this *HTTPWriter) Prepare(size int64, status int) {
this.statusCode = status
this.prepareGzip(size)
this.prepareCache(size)
}
// 包装前的原始的Writer
// Raw 包装前的原始的Writer
func (this *HTTPWriter) Raw() http.ResponseWriter {
return this.writer
}
// 获取Header
// Header 获取Header
func (this *HTTPWriter) Header() http.Header {
if this.writer == nil {
return http.Header{}
@@ -82,7 +87,7 @@ func (this *HTTPWriter) Header() http.Header {
return this.writer.Header()
}
// 添加一组Header
// AddHeaders 添加一组Header
func (this *HTTPWriter) AddHeaders(header http.Header) {
if this.writer == nil {
return
@@ -97,7 +102,7 @@ func (this *HTTPWriter) AddHeaders(header http.Header) {
}
}
// 写入数据
// Write 写入数据
func (this *HTTPWriter) Write(data []byte) (n int, err error) {
if this.writer != nil {
if this.gzipWriter != nil {
@@ -113,8 +118,9 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) {
if this.cacheWriter != nil {
_, err = this.cacheWriter.Write(data)
if err != nil {
_ = this.cacheWriter.Discard()
this.cacheWriter = nil
remotelogs.Error("REQUEST_WRITER", "write cache failed: "+err.Error())
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
}
}
} else {
@@ -126,7 +132,7 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) {
if this.gzipBodyWriter != nil {
_, err := this.gzipBodyWriter.Write(data)
if err != nil {
remotelogs.Error("REQUEST_WRITER", err.Error())
remotelogs.Error("HTTP_WRITER", err.Error())
}
} else {
this.body = append(this.body, data...)
@@ -135,17 +141,17 @@ func (this *HTTPWriter) Write(data []byte) (n int, err error) {
return
}
// 写入字符串
// WriteString 写入字符串
func (this *HTTPWriter) WriteString(s string) (n int, err error) {
return this.Write([]byte(s))
}
// 读取发送的字节数
// SentBodyBytes 读取发送的字节数
func (this *HTTPWriter) SentBodyBytes() int64 {
return this.sentBodyBytes
}
// 写入状态码
// WriteHeader 写入状态码
func (this *HTTPWriter) WriteHeader(statusCode int) {
if this.writer != nil {
this.writer.WriteHeader(statusCode)
@@ -153,7 +159,7 @@ func (this *HTTPWriter) WriteHeader(statusCode int) {
this.statusCode = statusCode
}
// 读取状态码
// StatusCode 读取状态码
func (this *HTTPWriter) StatusCode() int {
if this.statusCode == 0 {
return http.StatusOK
@@ -161,22 +167,22 @@ func (this *HTTPWriter) StatusCode() int {
return this.statusCode
}
// 设置拷贝Body数据
// SetBodyCopying 设置拷贝Body数据
func (this *HTTPWriter) SetBodyCopying(b bool) {
this.bodyCopying = b
}
// 判断是否在拷贝Body数据
// BodyIsCopying 判断是否在拷贝Body数据
func (this *HTTPWriter) BodyIsCopying() bool {
return this.bodyCopying
}
// 读取拷贝的Body数据
// Body 读取拷贝的Body数据
func (this *HTTPWriter) Body() []byte {
return this.body
}
// 读取Header二进制数据
// HeaderData 读取Header二进制数据
func (this *HTTPWriter) HeaderData() []byte {
if this.writer == nil {
return nil
@@ -198,7 +204,12 @@ func (this *HTTPWriter) HeaderData() []byte {
return writer.Bytes()
}
// 关闭
// SetOk 设置成功
func (this *HTTPWriter) SetOk() {
this.isOk = true
}
// Close 关闭
func (this *HTTPWriter) Close() {
// gzip writer
if this.gzipWriter != nil {
@@ -212,20 +223,38 @@ func (this *HTTPWriter) Close() {
// cache writer
if this.cacheWriter != nil {
err := this.cacheWriter.Close()
if err == nil {
this.cacheStorage.AddToList(&caches.Item{
Type: this.cacheWriter.ItemType(),
Key: this.cacheWriter.Key(),
ExpiredAt: this.cacheWriter.ExpiredAt(),
HeaderSize: this.cacheWriter.HeaderSize(),
BodySize: this.cacheWriter.BodySize(),
})
if this.isOk {
// 对比Content-Length
contentLengthString := this.Header().Get("Content-Length")
if len(contentLengthString) > 0 {
contentLength := types.Int64(contentLengthString)
if contentLength != this.cacheWriter.BodySize() {
this.isOk = false
_ = this.cacheWriter.Discard()
}
}
if this.isOk {
err := this.cacheWriter.Close()
if err == nil {
this.cacheStorage.AddToList(&caches.Item{
Type: this.cacheWriter.ItemType(),
Key: this.cacheWriter.Key(),
ExpiredAt: this.cacheWriter.ExpiredAt(),
HeaderSize: this.cacheWriter.HeaderSize(),
BodySize: this.cacheWriter.BodySize(),
Host: this.req.Host,
ServerId: this.req.Server.Id,
})
}
}
} else {
_ = this.cacheWriter.Discard()
}
}
}
// Hijack
// Hijack Hijack
func (this *HTTPWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
hijack, ok := this.writer.(http.Hijacker)
if ok {
@@ -234,7 +263,7 @@ func (this *HTTPWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err erro
return
}
// Flush
// Flush Flush
func (this *HTTPWriter) Flush() {
flusher, ok := this.writer.(http.Flusher)
if ok {
@@ -282,7 +311,7 @@ func (this *HTTPWriter) prepareGzip(size int64) {
var err error = nil
this.gzipWriter, err = gzip.NewWriterLevel(this.writer, int(this.gzipConfig.Level))
if err != nil {
remotelogs.Error("REQUEST_WRITER", err.Error())
remotelogs.Error("HTTP_WRITER", err.Error())
return
}
@@ -291,7 +320,7 @@ func (this *HTTPWriter) prepareGzip(size int64) {
this.gzipBodyBuffer = bytes.NewBuffer([]byte{})
this.gzipBodyWriter, err = gzip.NewWriterLevel(this.gzipBodyBuffer, int(this.gzipConfig.Level))
if err != nil {
remotelogs.Error("REQUEST_WRITER", err.Error())
remotelogs.Error("HTTP_WRITER", err.Error())
}
}
@@ -373,8 +402,8 @@ func (this *HTTPWriter) prepareCache(size int64) {
expiredAt := utils.UnixTime() + life
cacheWriter, err := storage.OpenWriter(this.req.cacheKey, expiredAt, this.StatusCode())
if err != nil {
if err != caches.ErrFileIsWriting {
remotelogs.Error("REQUEST_WRITER", "write cache failed: "+err.Error())
if !caches.CanIgnoreErr(err) {
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
}
return
}
@@ -388,7 +417,8 @@ func (this *HTTPWriter) prepareCache(size int64) {
for _, v1 := range v {
_, err = cacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
remotelogs.Error("REQUEST_WRITER", "write cache failed: "+err.Error())
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
_ = this.cacheWriter.Discard()
this.cacheWriter = nil
return
}

View File

@@ -0,0 +1,63 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"bufio"
"net"
"net/http"
)
// EmptyResponseWriter 空的响应Writer
type EmptyResponseWriter struct {
header http.Header
parentWriter http.ResponseWriter
statusCode int
}
func NewEmptyResponseWriter(parentWriter http.ResponseWriter) *EmptyResponseWriter {
return &EmptyResponseWriter{
header: http.Header{},
parentWriter: parentWriter,
}
}
func (this *EmptyResponseWriter) Header() http.Header {
return this.header
}
func (this *EmptyResponseWriter) Write(data []byte) (int, error) {
if this.statusCode > 300 && this.parentWriter != nil {
return this.parentWriter.Write(data)
}
return 0, nil
}
func (this *EmptyResponseWriter) WriteHeader(statusCode int) {
this.statusCode = statusCode
if this.statusCode > 300 && this.parentWriter != nil {
var parentHeader = this.parentWriter.Header()
for k, v := range this.header {
parentHeader[k] = v
}
this.parentWriter.WriteHeader(this.statusCode)
}
}
func (this *EmptyResponseWriter) StatusCode() int {
return this.statusCode
}
// Hijack Hijack
func (this *EmptyResponseWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
if this.parentWriter == nil {
return
}
hijack, ok := this.parentWriter.(http.Hijacker)
if ok {
return hijack.Hijack()
}
return
}

View File

@@ -43,8 +43,19 @@ func (this *Listener) Listen() error {
return nil
}
protocol := this.group.Protocol()
if protocol.IsUDPFamily() {
return this.listenUDP()
}
return this.listenTCP()
}
netListener, err := this.createListener()
func (this *Listener) listenTCP() error {
if this.group == nil {
return nil
}
protocol := this.group.Protocol()
netListener, err := this.createTCPListener()
if err != nil {
return err
}
@@ -80,11 +91,6 @@ func (this *Listener) Listen() error {
BaseListener: BaseListener{Group: this.group},
Listener: netListener,
}
case serverconfigs.ProtocolUDP:
this.listener = &UDPListener{
BaseListener: BaseListener{Group: this.group},
Listener: netListener,
}
default:
return errors.New("unknown protocol '" + protocol.String() + "'")
}
@@ -108,6 +114,31 @@ func (this *Listener) Listen() error {
return nil
}
func (this *Listener) listenUDP() error {
listener, err := this.createUDPListener()
if err != nil {
return err
}
events.On(events.EventQuit, func() {
remotelogs.Println("LISTENER", "quit "+this.group.FullAddr())
_ = listener.Close()
})
this.listener = &UDPListener{
BaseListener: BaseListener{Group: this.group},
Listener: listener,
}
go func() {
err := this.listener.Serve()
if err != nil {
remotelogs.Error("LISTENER", err.Error())
}
}()
return nil
}
func (this *Listener) Close() error {
if this.listener == nil {
return nil
@@ -115,8 +146,8 @@ func (this *Listener) Close() error {
return this.listener.Close()
}
// 创建监听器
func (this *Listener) createListener() (net.Listener, error) {
// 创建TCP监听器
func (this *Listener) createTCPListener() (net.Listener, error) {
listenConfig := net.ListenConfig{
Control: nil,
KeepAlive: 0,
@@ -131,3 +162,13 @@ func (this *Listener) createListener() (net.Listener, error) {
return listenConfig.Listen(context.Background(), "tcp", this.group.Addr())
}
// 创建UDP监听器
func (this *Listener) createUDPListener() (*net.UDPConn, error) {
// TODO 将来支持udp4/udp6
addr, err := net.ResolveUDPAddr("udp", this.group.Addr())
if err != nil {
return nil, err
}
return net.ListenUDP("udp", addr)
}

View File

@@ -7,7 +7,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
http2 "golang.org/x/net/http2"
"golang.org/x/net/http2"
"sync"
)

View File

@@ -4,13 +4,20 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"golang.org/x/net/http2"
"io"
"log"
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
var httpErrorLogger = log.New(io.Discard, "", 0)
var metricNewConnMap = map[string]bool{} // remoteAddr => bool
var metricNewConnMapLocker = &sync.Mutex{}
type HTTPListener struct {
BaseListener
@@ -35,14 +42,27 @@ func (this *HTTPListener) Serve() error {
this.httpServer = &http.Server{
Addr: this.addr,
Handler: handler,
ReadHeaderTimeout: 3 * time.Second, // TODO 改成可以配置
ReadHeaderTimeout: 2 * time.Second, // TODO 改成可以配置
IdleTimeout: 2 * time.Minute, // TODO 改成可以配置
ErrorLog: httpErrorLogger,
ConnState: func(conn net.Conn, state http.ConnState) {
switch state {
case http.StateNew:
atomic.AddInt64(&this.countActiveConnections, 1)
// 为指标存储连接信息
if sharedNodeConfig.HasHTTPConnectionMetrics() {
metricNewConnMapLocker.Lock()
metricNewConnMap[conn.RemoteAddr().String()] = true
metricNewConnMapLocker.Unlock()
}
case http.StateClosed:
atomic.AddInt64(&this.countActiveConnections, -1)
// 移除指标存储连接信息
metricNewConnMapLocker.Lock()
delete(metricNewConnMap, conn.RemoteAddr().String())
metricNewConnMapLocker.Unlock()
}
},
}

View File

@@ -2,20 +2,20 @@ package nodes
import "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
// 各协议监听器的接口
// ListenerInterface 各协议监听器的接口
type ListenerInterface interface {
// 初始化
// Init 初始化
Init()
// 监听
// Serve 监听
Serve() error
// 关闭
// Close 关闭
Close() error
// 重载配置
// Reload 重载配置
Reload(serverGroup *serverconfigs.ServerGroup)
// 获取当前活跃的连接数
// CountActiveListeners 获取当前活跃的连接数
CountActiveListeners() int
}

View File

@@ -4,33 +4,56 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/lists"
"net/url"
"regexp"
"sync"
"time"
)
var sharedListenerManager = NewListenerManager()
// 端口监听管理器
// ListenerManager 端口监听管理器
type ListenerManager struct {
listenersMap map[string]*Listener // addr => *Listener
locker sync.Mutex
lastConfig *nodeconfigs.NodeConfig
retryListenerMap map[string]*Listener // 需要重试的监听器 addr => Listener
ticker *time.Ticker
}
// 获取新对象
// NewListenerManager 获取新对象
func NewListenerManager() *ListenerManager {
return &ListenerManager{
listenersMap: map[string]*Listener{},
manager := &ListenerManager{
listenersMap: map[string]*Listener{},
retryListenerMap: map[string]*Listener{},
ticker: time.NewTicker(1 * time.Minute),
}
// 提升测试效率
if Tea.IsTesting() {
manager.ticker = time.NewTicker(5 * time.Second)
}
go func() {
for range manager.ticker.C {
manager.retryListeners()
}
}()
return manager
}
// 启动监听
// Start 启动监听
func (this *ListenerManager) Start(node *nodeconfigs.NodeConfig) error {
this.locker.Lock()
defer this.locker.Unlock()
// 重置数据
this.retryListenerMap = map[string]*Listener{}
// 检查是否有变化
/**if this.lastConfig != nil && this.lastConfig.Version == node.Version {
return nil
@@ -83,7 +106,16 @@ func (this *ListenerManager) Start(node *nodeconfigs.NodeConfig) error {
listener.Reload(group)
err := listener.Listen()
if err != nil {
remotelogs.Error("LISTENER_MANAGER", err.Error())
// 放入到重试队列中
this.retryListenerMap[addr] = listener
firstServer := group.FirstServer()
if firstServer == nil {
remotelogs.Error("LISTENER_MANAGER", err.Error())
} else {
remotelogs.ServerError(firstServer.Id, "LISTENER_MANAGER", err.Error())
}
continue
}
this.listenersMap[addr] = listener
@@ -93,7 +125,7 @@ func (this *ListenerManager) Start(node *nodeconfigs.NodeConfig) error {
return nil
}
// 获取总的活跃连接数
// TotalActiveConnections 获取总的活跃连接数
func (this *ListenerManager) TotalActiveConnections() int {
this.locker.Lock()
defer this.locker.Unlock()
@@ -116,3 +148,18 @@ func (this *ListenerManager) prettyAddress(addr string) string {
}
return u.String()
}
// 重试失败的Listener
func (this *ListenerManager) retryListeners() {
this.locker.Lock()
defer this.locker.Unlock()
for addr, listener := range this.retryListenerMap {
err := listener.Listen()
if err == nil {
delete(this.retryListenerMap, addr)
this.listenersMap[addr] = listener
remotelogs.ServerSuccess(listener.group.FirstServer().Id, "LISTENER_MANAGER", "retry to listen '"+addr+"' successfully")
}
}
}

View File

@@ -80,7 +80,7 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
}
// 记录流量
stats.SharedTrafficStatManager.Add(firstServer.Id, int64(n))
stats.SharedTrafficStatManager.Add(firstServer.Id, "", int64(n), 0, 0, 0, 0, 0)
}
if err != nil {
closer()

View File

@@ -1,28 +1,201 @@
package nodes
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"net"
"sync"
"time"
)
type UDPListener struct {
BaseListener
Listener net.Listener
Listener *net.UDPConn
connMap map[string]*UDPConn
connLocker sync.Mutex
connTicker *utils.Ticker
}
func (this *UDPListener) Serve() error {
// TODO
// TODO 注意管理 CountActiveConnections
return nil
firstServer := this.Group.FirstServer()
if firstServer == nil {
return errors.New("no server available")
}
if firstServer.ReverseProxy == nil {
return errors.New("no ReverseProxy configured for the server")
}
this.connMap = map[string]*UDPConn{}
this.connTicker = utils.NewTicker(1 * time.Minute)
go func() {
for this.connTicker.Next() {
this.gcConns()
}
}()
var buffer = make([]byte, 4*1024)
for {
n, addr, _ := this.Listener.ReadFrom(buffer)
if n > 0 {
this.connLocker.Lock()
conn, ok := this.connMap[addr.String()]
this.connLocker.Unlock()
if ok && !conn.IsOk() {
_ = conn.Close()
ok = false
}
if !ok {
originConn, err := this.connectOrigin(firstServer.ReverseProxy, "")
if err != nil {
remotelogs.Error("UDP_LISTENER", "unable to connect to origin server: "+err.Error())
continue
}
if originConn == nil {
remotelogs.Error("UDP_LISTENER", "unable to find a origin server")
continue
}
conn = NewUDPConn(firstServer.Id, addr, this.Listener, originConn.(*net.UDPConn))
this.connLocker.Lock()
this.connMap[addr.String()] = conn
this.connLocker.Unlock()
}
_, _ = conn.Write(buffer[:n])
}
}
}
func (this *UDPListener) Close() error {
// TODO
return nil
if this.connTicker != nil {
this.connTicker.Stop()
}
// 关闭所有连接
this.connLocker.Lock()
for _, conn := range this.connMap {
_ = conn.Close()
}
this.connLocker.Unlock()
return this.Listener.Close()
}
func (this *UDPListener) Reload(group *serverconfigs.ServerGroup) {
this.Group = group
this.Reset()
}
func (this *UDPListener) connectOrigin(reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
if reverseProxy == nil {
return nil, errors.New("no reverse proxy config")
}
retries := 3
for i := 0; i < retries; i++ {
origin := reverseProxy.NextOrigin(nil)
if origin == nil {
continue
}
conn, err = OriginConnect(origin, remoteAddr)
if err != nil {
remotelogs.Error("UDP_LISTENER", "unable to connect origin: "+origin.Addr.Host+":"+origin.Addr.PortRange+": "+err.Error())
continue
} else {
return
}
}
err = errors.New("no origin can be used")
return
}
// 回收连接
func (this *UDPListener) gcConns() {
this.connLocker.Lock()
closingConns := []*UDPConn{}
for addr, conn := range this.connMap {
if !conn.IsOk() {
closingConns = append(closingConns, conn)
delete(this.connMap, addr)
}
}
this.connLocker.Unlock()
for _, conn := range closingConns {
_ = conn.Close()
}
}
// UDPConn 自定义的UDP连接管理
type UDPConn struct {
addr net.Addr
proxyConn net.Conn
serverConn net.Conn
activatedAt int64
isOk bool
isClosed bool
}
func NewUDPConn(serverId int64, addr net.Addr, proxyConn *net.UDPConn, serverConn *net.UDPConn) *UDPConn {
conn := &UDPConn{
addr: addr,
proxyConn: proxyConn,
serverConn: serverConn,
activatedAt: time.Now().Unix(),
isOk: true,
}
go func() {
buffer := bytePool32k.Get()
defer func() {
bytePool32k.Put(buffer)
}()
for {
n, err := serverConn.Read(buffer)
if n > 0 {
conn.activatedAt = time.Now().Unix()
_, writingErr := proxyConn.WriteTo(buffer[:n], addr)
if writingErr != nil {
conn.isOk = false
break
}
// 记录流量
stats.SharedTrafficStatManager.Add(serverId, "", int64(n), 0, 0, 0, 0, 0)
}
if err != nil {
conn.isOk = false
break
}
}
}()
return conn
}
func (this *UDPConn) Write(b []byte) (n int, err error) {
this.activatedAt = time.Now().Unix()
n, err = this.serverConn.Write(b)
if err != nil {
this.isOk = false
}
return
}
func (this *UDPConn) Close() error {
this.isOk = false
if this.isClosed {
return nil
}
this.isClosed = true
return this.serverConn.Close()
}
func (this *UDPConn) IsOk() bool {
if !this.isOk {
return false
}
return time.Now().Unix()-this.activatedAt < 30 // 如果超过 N 秒没有活动我们认为是超时
}

View File

@@ -12,6 +12,7 @@ import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/stats"
@@ -33,8 +34,10 @@ import (
var sharedNodeConfig *nodeconfigs.NodeConfig
var nodeTaskNotify = make(chan bool, 8)
var DaemonIsOn = false
var DaemonPid = 0
// 节点
// Node 节点
type Node struct {
isLoaded bool
}
@@ -43,7 +46,7 @@ func NewNode() *Node {
return &Node{}
}
// 检查配置
// Test 检查配置
func (this *Node) Test() error {
// 检查是否能连接API
rpcClient, err := rpc.SharedRPC()
@@ -58,8 +61,15 @@ func (this *Node) Test() error {
return nil
}
// 启动
// Start 启动
func (this *Node) Start() {
_, ok := os.LookupEnv("EdgeDaemon")
if ok {
remotelogs.Println("NODE", "start from daemon")
DaemonIsOn = true
DaemonPid = os.Getppid()
}
// 启动事件
events.Notify(events.EventStart)
@@ -74,23 +84,30 @@ func (this *Node) Start() {
}
// 读取API配置
tryTimes := 0
for {
err = this.syncConfig()
err = this.syncConfig()
if err != nil {
_, err := nodeconfigs.SharedNodeConfig()
if err != nil {
tryTimes++
// 无本地数据时,会尝试多次读取
tryTimes := 0
for {
err := this.syncConfig()
if err != nil {
tryTimes++
if tryTimes%10 == 0 {
remotelogs.Error("NODE", err.Error())
}
time.Sleep(1 * time.Second)
if tryTimes%10 == 0 {
remotelogs.Error("NODE", err.Error())
}
time.Sleep(1 * time.Second)
// 不做长时间的无意义的重试
if tryTimes > 1000 {
return
// 不做长时间的无意义的重试
if tryTimes > 1000 {
return
}
} else {
break
}
}
} else {
break
}
}
@@ -146,7 +163,7 @@ func (this *Node) Start() {
select {}
}
// 实现守护进程
// Daemon 实现守护进程
func (this *Node) Daemon() {
path := os.TempDir() + "/edge-node.sock"
isDebug := lists.ContainsString(os.Args, "debug")
@@ -164,6 +181,10 @@ func (this *Node) Daemon() {
if err != nil {
return err
}
// 可以标记当前是从守护进程启动的
_ = os.Setenv("EdgeDaemon", "on")
cmd := exec.Command(exe)
err = cmd.Start()
if err != nil {
@@ -191,7 +212,7 @@ func (this *Node) Daemon() {
}
}
// 安装系统服务
// InstallSystemService 安装系统服务
func (this *Node) InstallSystemService() error {
shortName := teaconst.SystemdServiceName
@@ -285,6 +306,8 @@ func (this *Node) loop() error {
if err != nil {
return err
}
case "nodeVersionChanged":
go sharedUpgradeManager.Start()
}
}
@@ -374,6 +397,8 @@ func (this *Node) syncConfig() error {
iplibrary.SharedActionManager.UpdateActions(nodeConfig.FirewallActions)
sharedNodeConfig = nodeConfig
metrics.SharedManager.Update(nodeConfig.MetricItems)
// 发送事件
events.Notify(events.EventReload)

View File

@@ -15,6 +15,7 @@ import (
"github.com/iwind/TeaGo/maps"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/disk"
"golang.org/x/sys/unix"
"os"
"runtime"
"strings"
@@ -67,6 +68,8 @@ func (this *NodeStatusExecutor) update() {
status.ConnectionCount = sharedListenerManager.TotalActiveConnections()
status.CacheTotalDiskSize = caches.SharedManager.TotalDiskSize()
status.CacheTotalMemorySize = caches.SharedManager.TotalMemorySize()
status.TrafficInBytes = inTrafficBytes
status.TrafficOutBytes = outTrafficBytes
// 记录监控数据
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemConnections, maps.Map{
@@ -80,6 +83,7 @@ func (this *NodeStatusExecutor) update() {
this.updateMem(status)
this.updateLoad(status)
this.updateDisk(status)
this.updateCacheSpace(status)
status.UpdatedAt = time.Now().Unix()
// 发送数据
@@ -211,3 +215,25 @@ func (this *NodeStatusExecutor) updateDisk(status *nodeconfigs.NodeStatus) {
"maxUsage": status.DiskMaxUsage,
})
}
// 缓存空间
func (this *NodeStatusExecutor) updateCacheSpace(status *nodeconfigs.NodeStatus) {
var result = []maps.Map{}
cachePaths := caches.SharedManager.FindAllCachePaths()
for _, path := range cachePaths {
var stat unix.Statfs_t
err := unix.Statfs(path, &stat)
if err != nil {
return
}
result = append(result, maps.Map{
"path": path,
"total": stat.Blocks * uint64(stat.Bsize),
"avail": stat.Bavail * uint64(stat.Bsize),
"used": (stat.Blocks - stat.Bavail) * uint64(stat.Bsize),
})
}
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemCacheDir, maps.Map{
"dirs": result,
})
}

View File

@@ -9,7 +9,7 @@ import (
"strconv"
)
// 连接源站
// OriginConnect 连接源站
func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.Conn, error) {
if origin.Addr == nil {
return nil, errors.New("origin server address should not be empty")
@@ -70,9 +70,15 @@ func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.C
return tls.Dial("tcp", origin.Addr.Host+":"+origin.Addr.PortRange, &tls.Config{
InsecureSkipVerify: true,
})
case serverconfigs.ProtocolUDP:
addr, err := net.ResolveUDPAddr("udp", origin.Addr.Host+":"+origin.Addr.PortRange)
if err != nil {
return nil, err
}
return net.DialUDP("udp", nil, addr)
}
// TODO 支持从Unix、Pipe、HTTP、HTTPS中读取数据
return nil, errors.New("invalid scheme '" + origin.Addr.Protocol.String() + "'")
return nil, errors.New("invalid origin scheme '" + origin.Addr.Protocol.String() + "'")
}

View File

@@ -1,6 +1,8 @@
package nodes
import (
"context"
"crypto/tls"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/configs"
"github.com/TeaOSLab/EdgeNode/internal/events"
@@ -8,8 +10,12 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"net/url"
"sort"
"strings"
"sync"
"time"
)
@@ -20,7 +26,7 @@ func init() {
})
}
// API节点同步任务
// SyncAPINodesTask API节点同步任务
type SyncAPINodesTask struct {
}
@@ -74,6 +80,12 @@ func (this *SyncAPINodesTask) Loop() error {
return nil
}
// 测试是否有API节点可用
hasOk := this.testEndpoints(newEndpoints)
if !hasOk {
return nil
}
// 修改RPC对象配置
config.RPC.Endpoints = newEndpoints
err = rpcClient.UpdateConfig(config)
@@ -95,3 +107,47 @@ func (this *SyncAPINodesTask) isSame(endpoints1 []string, endpoints2 []string) b
sort.Strings(endpoints2)
return strings.Join(endpoints1, "&") == strings.Join(endpoints2, "&")
}
func (this *SyncAPINodesTask) testEndpoints(endpoints []string) bool {
if len(endpoints) == 0 {
return false
}
var wg = sync.WaitGroup{}
wg.Add(len(endpoints))
var ok = false
for _, endpoint := range endpoints {
go func(endpoint string) {
defer wg.Done()
u, err := url.Parse(endpoint)
if err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer func() {
cancel()
}()
var conn *grpc.ClientConn
if u.Scheme == "http" {
conn, err = grpc.DialContext(ctx, u.Host, grpc.WithInsecure(), grpc.WithBlock())
} else if u.Scheme == "https" {
conn, err = grpc.DialContext(ctx, u.Host, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})), grpc.WithBlock())
}
if err != nil {
return
}
_ = conn.Close()
ok = true
}(endpoint)
}
wg.Wait()
return ok
}

View File

@@ -86,6 +86,7 @@ func (this *TOAManager) SendMsg(msg string) error {
if this.conn != nil {
_, err := this.conn.Write([]byte(msg + "\n"))
if err != nil {
_ = this.conn.Close()
this.conn = nil
}
return err

View File

@@ -2,7 +2,10 @@
package nodes
import "net"
import (
"github.com/TeaOSLab/EdgeNode/internal/waf"
"net"
)
// TrafficListener 用于统计流量的网络监听
type TrafficListener struct {
@@ -18,6 +21,17 @@ func (this *TrafficListener) Accept() (net.Conn, error) {
if err != nil {
return nil, err
}
// 是否在WAF名单中
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err == nil {
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, ip) && waf.SharedIPBlackLIst.Contains(waf.IPTypeAll, ip) {
go func() {
_ = conn.Close()
}()
return conn, nil
}
}
return NewTrafficConn(conn), nil
}

View File

@@ -0,0 +1,252 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"crypto/md5"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
stringutil "github.com/iwind/TeaGo/utils/string"
"os"
"os/exec"
"runtime"
"time"
)
var sharedUpgradeManager = NewUpgradeManager()
// UpgradeManager 节点升级管理器
// TODO 需要在集群中设置是否自动更新
type UpgradeManager struct {
isInstalling bool
lastFile string
}
// NewUpgradeManager 获取新对象
func NewUpgradeManager() *UpgradeManager {
return &UpgradeManager{}
}
// Start 启动升级
func (this *UpgradeManager) Start() {
// 测试环境下不更新
if Tea.IsTesting() {
return
}
if this.isInstalling {
return
}
this.isInstalling = true
// 还原安装状态
defer func() {
this.isInstalling = false
}()
remotelogs.Println("UPGRADE_MANAGER", "upgrading node ...")
err := this.install()
if err != nil {
remotelogs.Error("UPGRADE_MANAGER", "download failed: "+err.Error())
return
}
remotelogs.Println("UPGRADE_MANAGER", "upgrade successfully")
go func() {
err = this.restart()
if err != nil {
logs.Println("UPGRADE_MANAGER", err.Error())
}
}()
}
func (this *UpgradeManager) install() error {
// 检查是否有已下载但未安装成功的
if len(this.lastFile) > 0 {
_, err := os.Stat(this.lastFile)
if err == nil {
err = this.unzip(this.lastFile)
if err != nil {
return err
}
this.lastFile = ""
return nil
}
}
// 创建临时文件
dir := Tea.Root + "/tmp"
_, err := os.Stat(dir)
if err != nil {
if os.IsNotExist(err) {
err = os.Mkdir(dir, 0777)
if err != nil {
return err
}
} else {
return err
}
}
remotelogs.Println("UPGRADE_MANAGER", "downloading new node ...")
path := dir + "/edge-node" + ".tmp"
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0777)
if err != nil {
return err
}
isClosed := false
defer func() {
if !isClosed {
_ = fp.Close()
}
}()
client, err := rpc.SharedRPC()
if err != nil {
return err
}
var offset int64
var h = md5.New()
var sum = ""
var filename = ""
for {
resp, err := client.NodeRPC().DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
Os: runtime.GOOS,
Arch: runtime.GOARCH,
ChunkOffset: offset,
})
if err != nil {
return err
}
if len(resp.Sum) == 0 {
return nil
}
sum = resp.Sum
filename = resp.Filename
if stringutil.VersionCompare(resp.Version, teaconst.Version) <= 0 {
return nil
}
if len(resp.ChunkData) == 0 {
break
}
// 写入文件
_, err = fp.Write(resp.ChunkData)
if err != nil {
return err
}
_, err = h.Write(resp.ChunkData)
if err != nil {
return err
}
offset = resp.Offset
}
if len(filename) == 0 {
return nil
}
isClosed = true
err = fp.Close()
if err != nil {
return err
}
if fmt.Sprintf("%x", h.Sum(nil)) != sum {
_ = os.Remove(path)
return nil
}
// 改成zip
zipPath := dir + "/" + filename
err = os.Rename(path, zipPath)
if err != nil {
return err
}
this.lastFile = zipPath
// 解压
err = this.unzip(zipPath)
if err != nil {
return err
}
return nil
}
// 解压
func (this *UpgradeManager) unzip(zipPath string) error {
var isOk = false
defer func() {
if isOk {
// 只有解压并覆盖成功后才会删除
_ = os.Remove(zipPath)
}
}()
// 解压
var target = Tea.Root
if Tea.IsTesting() {
// 测试环境下只解压在tmp目录
target = Tea.Root + "/tmp"
}
// 先改先前的可执行文件
err := os.Rename(target+"/bin/edge-node", target+"/bin/.edge-node.old")
hasBackup := err == nil
defer func() {
if !isOk && hasBackup {
// 失败时还原
_ = os.Rename(target+"/bin/.edge-node.old", target+"/bin/edge-node")
}
}()
unzip := utils.NewUnzip(zipPath, target, "edge-node/")
err = unzip.Run()
if err != nil {
return err
}
isOk = true
return nil
}
// 重启
func (this *UpgradeManager) restart() error {
// 重新启动
if DaemonIsOn && DaemonPid == os.Getppid() {
os.Exit(0) // TODO 试着更优雅重启
} else {
exe, err := os.Executable()
if err != nil {
return err
}
// quit
events.Notify(events.EventQuit)
// 启动
cmd := exec.Command(exe, "start")
err = cmd.Start()
if err != nil {
return err
}
// 退出当前进程
time.Sleep(1 * time.Second)
os.Exit(0)
}
return nil
}

View File

@@ -0,0 +1,16 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
_ "github.com/iwind/TeaGo/bootstrap"
"testing"
)
func TestUpgradeManager_install(t *testing.T) {
err := NewUpgradeManager().install()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -11,20 +11,20 @@ import (
var sharedWAFManager = NewWAFManager()
// WAF管理器
// WAFManager WAF管理器
type WAFManager struct {
mapping map[int64]*waf.WAF // policyId => WAF
locker sync.RWMutex
}
// 获取新对象
// NewWAFManager 获取新对象
func NewWAFManager() *WAFManager {
return &WAFManager{
mapping: map[int64]*waf.WAF{},
}
}
// 更新策略
// UpdatePolicies 更新策略
func (this *WAFManager) UpdatePolicies(policies []*firewallconfigs.HTTPFirewallPolicy) {
this.locker.Lock()
defer this.locker.Unlock()
@@ -44,7 +44,7 @@ func (this *WAFManager) UpdatePolicies(policies []*firewallconfigs.HTTPFirewallP
this.mapping = m
}
// 查找WAF
// FindWAF 查找WAF
func (this *WAFManager) FindWAF(policyId int64) *waf.WAF {
this.locker.RLock()
w, _ := this.mapping[policyId]
@@ -78,14 +78,15 @@ func (this *WAFManager) convertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
// rule sets
for _, set := range group.Sets {
s := &waf.RuleSet{
Id: strconv.FormatInt(set.Id, 10),
Code: set.Code,
IsOn: set.IsOn,
Name: set.Name,
Description: set.Description,
Connector: set.Connector,
Action: set.Action,
ActionOptions: set.ActionOptions,
Id: strconv.FormatInt(set.Id, 10),
Code: set.Code,
IsOn: set.IsOn,
Name: set.Name,
Description: set.Description,
Connector: set.Connector,
}
for _, a := range set.Actions {
s.AddAction(a.Code, a.Options)
}
// rules
@@ -132,14 +133,16 @@ func (this *WAFManager) convertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
// rule sets
for _, set := range group.Sets {
s := &waf.RuleSet{
Id: strconv.FormatInt(set.Id, 10),
Code: set.Code,
IsOn: set.IsOn,
Name: set.Name,
Description: set.Description,
Connector: set.Connector,
Action: set.Action,
ActionOptions: set.ActionOptions,
Id: strconv.FormatInt(set.Id, 10),
Code: set.Code,
IsOn: set.IsOn,
Name: set.Name,
Description: set.Description,
Connector: set.Connector,
}
for _, a := range set.Actions {
s.AddAction(a.Code, a.Options)
}
// rules
@@ -164,10 +167,11 @@ func (this *WAFManager) convertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
// action
if policy.BlockOptions != nil {
w.ActionBlock = &waf.BlockAction{
w.DefaultBlockAction = &waf.BlockAction{
StatusCode: policy.BlockOptions.StatusCode,
Body: policy.BlockOptions.Body,
URL: "",
URL: policy.BlockOptions.URL,
Timeout: policy.BlockOptions.Timeout,
}
}

View File

@@ -24,7 +24,7 @@ func init() {
}()
}
// 打印普通信息
// Println 打印普通信息
func Println(tag string, description string) {
logs.Println("[" + tag + "]" + description)
@@ -47,7 +47,7 @@ func Println(tag string, description string) {
}
}
// 打印警告信息
// Warn 打印警告信息
func Warn(tag string, description string) {
logs.Println("[" + tag + "]" + description)
@@ -70,7 +70,7 @@ func Warn(tag string, description string) {
}
}
// 打印错误信息
// Error 打印错误信息
func Error(tag string, description string) {
logs.Println("[" + tag + "]" + description)
@@ -93,6 +93,54 @@ func Error(tag string, description string) {
}
}
// ServerError 打印服务相关错误信息
func ServerError(serverId int64, tag string, description string) {
logs.Println("[" + tag + "]" + description)
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig == nil {
return
}
select {
case logChan <- &pb.NodeLog{
Role: teaconst.Role,
Tag: tag,
Description: description,
Level: "error",
NodeId: nodeConfig.Id,
ServerId: serverId,
CreatedAt: time.Now().Unix(),
}:
default:
}
}
// ServerSuccess 打印服务相关成功信息
func ServerSuccess(serverId int64, tag string, description string) {
logs.Println("[" + tag + "]" + description)
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig == nil {
return
}
select {
case logChan <- &pb.NodeLog{
Role: teaconst.Role,
Tag: tag,
Description: description,
Level: "success",
NodeId: nodeConfig.Id,
ServerId: serverId,
CreatedAt: time.Now().Unix(),
}:
default:
}
}
// 上传日志
func uploadLogs() error {
logList := []*pb.NodeLog{}

View File

@@ -109,6 +109,14 @@ func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient {
return pb.NewServerDailyStatServiceClient(this.pickConn())
}
func (this *RPCClient) MetricStatRPC() pb.MetricStatServiceClient {
return pb.NewMetricStatServiceClient(this.pickConn())
}
func (this *RPCClient) FirewallService() pb.FirewallServiceClient {
return pb.NewFirewallServiceClient(this.pickConn())
}
// Context 节点上下文信息
func (this *RPCClient) Context() context.Context {
ctx := context.Background()

View File

@@ -1,12 +1,16 @@
package stats
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/monitor"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"github.com/mssola/user_agent"
@@ -17,7 +21,7 @@ import (
var SharedHTTPRequestStatManager = NewHTTPRequestStatManager()
// HTTP请求相关的统计
// HTTPRequestStatManager HTTP请求相关的统计
// 这里的统计是一个辅助统计,注意不要因为统计而影响服务工作性能
type HTTPRequestStatManager struct {
ipChan chan string
@@ -30,24 +34,39 @@ type HTTPRequestStatManager struct {
browserMap map[string]int64 // serverId@browser@version => count
dailyFirewallRuleGroupMap map[string]int64 // serverId@firewallRuleGroupId@action => count
totalAttackRequests int64
}
// 获取新对象
// NewHTTPRequestStatManager 获取新对象
func NewHTTPRequestStatManager() *HTTPRequestStatManager {
return &HTTPRequestStatManager{
ipChan: make(chan string, 10_000), // TODO 将来可以配置容量
userAgentChan: make(chan string, 10_000), // TODO 将来可以配置容量
firewallRuleGroupChan: make(chan string, 10_000), // TODO 将来可以配置容量
cityMap: map[string]int64{},
providerMap: map[string]int64{},
systemMap: map[string]int64{},
browserMap: map[string]int64{},
cityMap: map[string]int64{},
providerMap: map[string]int64{},
systemMap: map[string]int64{},
browserMap: map[string]int64{},
dailyFirewallRuleGroupMap: map[string]int64{},
}
}
// 启动
// Start 启动
func (this *HTTPRequestStatManager) Start() {
// 上传请求总数
go func() {
ticker := time.NewTicker(1 * time.Minute)
go func() {
for range ticker.C {
if this.totalAttackRequests > 0 {
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemAttackRequests, maps.Map{"total": this.totalAttackRequests})
this.totalAttackRequests = 0
}
}
}()
}()
loopTicker := time.NewTicker(1 * time.Second)
uploadTicker := time.NewTicker(30 * time.Minute)
if Tea.IsTesting() {
@@ -76,7 +95,7 @@ func (this *HTTPRequestStatManager) Start() {
}
}
// 添加客户端地址
// AddRemoteAddr 添加客户端地址
func (this *HTTPRequestStatManager) AddRemoteAddr(serverId int64, remoteAddr string) {
if len(remoteAddr) == 0 {
return
@@ -100,7 +119,7 @@ func (this *HTTPRequestStatManager) AddRemoteAddr(serverId int64, remoteAddr str
}
}
// 添加UserAgent
// AddUserAgent 添加UserAgent
func (this *HTTPRequestStatManager) AddUserAgent(serverId int64, userAgent string) {
if len(userAgent) == 0 {
return
@@ -113,19 +132,24 @@ func (this *HTTPRequestStatManager) AddUserAgent(serverId int64, userAgent strin
}
}
// 添加防火墙拦截动作
func (this *HTTPRequestStatManager) AddFirewallRuleGroupId(serverId int64, firewallRuleGroupId int64, action string) {
// AddFirewallRuleGroupId 添加防火墙拦截动作
func (this *HTTPRequestStatManager) AddFirewallRuleGroupId(serverId int64, firewallRuleGroupId int64, actions []*waf.ActionConfig) {
if firewallRuleGroupId <= 0 {
return
}
select {
case this.firewallRuleGroupChan <- strconv.FormatInt(serverId, 10) + "@" + strconv.FormatInt(firewallRuleGroupId, 10) + "@" + action:
default:
// 超出容量我们就丢弃
this.totalAttackRequests++
for _, action := range actions {
select {
case this.firewallRuleGroupChan <- strconv.FormatInt(serverId, 10) + "@" + strconv.FormatInt(firewallRuleGroupId, 10) + "@" + action.Code:
default:
// 超出容量我们就丢弃
}
}
}
// 单个循环
// Loop 单个循环
func (this *HTTPRequestStatManager) Loop() error {
timeout := time.NewTimer(10 * time.Minute) // 执行的最大时间
userAgentParser := &user_agent.UserAgent{}
@@ -189,6 +213,7 @@ Loop:
return nil
}
// Upload 上传数据
func (this *HTTPRequestStatManager) Upload() error {
// 上传统计数据
rpcClient, err := rpc.SharedRPC()

View File

@@ -4,28 +4,45 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/monitor"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"strconv"
"strings"
"sync"
"time"
)
var SharedTrafficStatManager = NewTrafficStatManager()
type TrafficItem struct {
Bytes int64
CachedBytes int64
CountRequests int64
CountCachedRequests int64
CountAttackRequests int64
AttackBytes int64
}
// TrafficStatManager 区域流量统计
type TrafficStatManager struct {
m map[string]int64 // [timestamp serverId] => bytes
itemMap map[string]*TrafficItem // [timestamp serverId] => *TrafficItem
domainsMap map[string]*TrafficItem // timestamp @ serverId @ domain => *TrafficItem
locker sync.Mutex
configFunc func() *nodeconfigs.NodeConfig
totalRequests int64
}
// NewTrafficStatManager 获取新对象
func NewTrafficStatManager() *TrafficStatManager {
manager := &TrafficStatManager{
m: map[string]int64{},
itemMap: map[string]*TrafficItem{},
domainsMap: map[string]*TrafficItem{},
}
return manager
@@ -35,6 +52,20 @@ func NewTrafficStatManager() *TrafficStatManager {
func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig) {
this.configFunc = configFunc
// 上传请求总数
go func() {
ticker := time.NewTicker(1 * time.Minute)
go func() {
for range ticker.C {
if this.totalRequests > 0 {
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemRequests, maps.Map{"total": this.totalRequests})
this.totalRequests = 0
}
}
}()
}()
// 上传统计数据
duration := 5 * time.Minute
if Tea.IsTesting() {
// 测试环境缩短上传时间,方便我们调试
@@ -55,16 +86,45 @@ func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig)
}
// Add 添加流量
func (this *TrafficStatManager) Add(serverId int64, bytes int64) {
func (this *TrafficStatManager) Add(serverId int64, domain string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64) {
if bytes == 0 {
return
}
this.totalRequests++
timestamp := utils.UnixTime() / 300 * 300
key := strconv.FormatInt(timestamp, 10) + strconv.FormatInt(serverId, 10)
this.locker.Lock()
this.m[key] += bytes
// 总的流量
item, ok := this.itemMap[key]
if !ok {
item = &TrafficItem{}
this.itemMap[key] = item
}
item.Bytes += bytes
item.CachedBytes += cachedBytes
item.CountRequests += countRequests
item.CountCachedRequests += countCachedRequests
item.CountAttackRequests += countAttacks
item.AttackBytes += attackBytes
// 单个域名流量
var domainKey = strconv.FormatInt(timestamp, 10) + "@" + strconv.FormatInt(serverId, 10) + "@" + domain
domainItem, ok := this.domainsMap[domainKey]
if !ok {
domainItem = &TrafficItem{}
this.domainsMap[domainKey] = domainItem
}
domainItem.Bytes += bytes
domainItem.CachedBytes += cachedBytes
domainItem.CountRequests += countRequests
domainItem.CountCachedRequests += countCachedRequests
domainItem.CountAttackRequests += countAttacks
domainItem.AttackBytes += attackBytes
this.locker.Unlock()
}
@@ -81,12 +141,15 @@ func (this *TrafficStatManager) Upload() error {
}
this.locker.Lock()
m := this.m
this.m = map[string]int64{}
itemMap := this.itemMap
domainMap := this.domainsMap
this.itemMap = map[string]*TrafficItem{}
this.domainsMap = map[string]*TrafficItem{}
this.locker.Unlock()
pbStats := []*pb.ServerDailyStat{}
for key, bytes := range m {
// 服务统计
var pbServerStats = []*pb.ServerDailyStat{}
for key, item := range itemMap {
timestamp, err := strconv.ParseInt(key[:10], 10, 64)
if err != nil {
return err
@@ -96,16 +159,45 @@ func (this *TrafficStatManager) Upload() error {
return err
}
pbStats = append(pbStats, &pb.ServerDailyStat{
ServerId: serverId,
RegionId: config.RegionId,
Bytes: bytes,
CreatedAt: timestamp,
pbServerStats = append(pbServerStats, &pb.ServerDailyStat{
ServerId: serverId,
RegionId: config.RegionId,
Bytes: item.Bytes,
CachedBytes: item.CachedBytes,
CountRequests: item.CountRequests,
CountCachedRequests: item.CountCachedRequests,
CountAttackRequests: item.CountAttackRequests,
AttackBytes: item.AttackBytes,
CreatedAt: timestamp,
})
}
if len(pbStats) == 0 {
if len(pbServerStats) == 0 {
return nil
}
_, err = client.ServerDailyStatRPC().UploadServerDailyStats(client.Context(), &pb.UploadServerDailyStatsRequest{Stats: pbStats})
// 域名统计
var pbDomainStats = []*pb.UploadServerDailyStatsRequest_DomainStat{}
for key, item := range domainMap {
var pieces = strings.SplitN(key, "@", 3)
if len(pieces) != 3 {
continue
}
pbDomainStats = append(pbDomainStats, &pb.UploadServerDailyStatsRequest_DomainStat{
ServerId: types.Int64(pieces[1]),
Domain: pieces[2],
Bytes: item.Bytes,
CachedBytes: item.CachedBytes,
CountRequests: item.CountRequests,
CountCachedRequests: item.CountCachedRequests,
CountAttackRequests: item.CountAttackRequests,
AttackBytes: item.AttackBytes,
CreatedAt: types.Int64(pieces[0]),
})
}
_, err = client.ServerDailyStatRPC().UploadServerDailyStats(client.Context(), &pb.UploadServerDailyStatsRequest{
Stats: pbServerStats,
DomainStats: pbDomainStats,
})
return err
}

View File

@@ -8,15 +8,15 @@ import (
func TestTrafficStatManager_Add(t *testing.T) {
manager := NewTrafficStatManager()
for i := 0; i < 100; i++ {
manager.Add(1, 10)
manager.Add(1, "goedge.cn", 1, 0, 0, 0)
}
t.Log(manager.m)
t.Log(manager.itemMap)
}
func TestTrafficStatManager_Upload(t *testing.T) {
manager := NewTrafficStatManager()
for i := 0; i < 100; i++ {
manager.Add(1, 10)
manager.Add(1, "goedge.cn", 1, 0, 0, 0)
}
err := manager.Upload()
if err != nil {
@@ -30,6 +30,6 @@ func BenchmarkTrafficStatManager_Add(b *testing.B) {
manager := NewTrafficStatManager()
for i := 0; i < b.N; i++ {
manager.Add(1, 1024)
manager.Add(1, "goedge.cn", 1024, 1, 0, 0)
}
}

View File

@@ -30,7 +30,7 @@ func (this *Piece) Add(key uint64, item *Item) () {
func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64) (result int64) {
this.locker.Lock()
item, ok := this.m[key]
if ok {
if ok && item.expiredAt > time.Now().Unix() {
result = types.Int64(item.Value) + delta
item.Value = result
item.expiredAt = expiredAt

159
internal/utils/encrypt.go Normal file
View File

@@ -0,0 +1,159 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package utils
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"encoding/base64"
"encoding/json"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
stringutil "github.com/iwind/TeaGo/utils/string"
)
var (
simpleEncryptMagicKey = rands.HexString(32)
)
func init() {
events.On(events.EventReload, func() {
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil {
simpleEncryptMagicKey = stringutil.Md5(nodeConfig.NodeId + "@" + nodeConfig.Secret)
}
})
}
// SimpleEncrypt 加密特殊信息
func SimpleEncrypt(data []byte) []byte {
var method = &AES256CFBMethod{}
err := method.Init([]byte(simpleEncryptMagicKey), []byte(simpleEncryptMagicKey[:16]))
if err != nil {
logs.Println("[SimpleEncrypt]" + err.Error())
return data
}
dst, err := method.Encrypt(data)
if err != nil {
logs.Println("[SimpleEncrypt]" + err.Error())
return data
}
return dst
}
// SimpleDecrypt 解密特殊信息
func SimpleDecrypt(data []byte) []byte {
var method = &AES256CFBMethod{}
err := method.Init([]byte(simpleEncryptMagicKey), []byte(simpleEncryptMagicKey[:16]))
if err != nil {
logs.Println("[MagicKeyEncode]" + err.Error())
return data
}
src, err := method.Decrypt(data)
if err != nil {
logs.Println("[MagicKeyEncode]" + err.Error())
return data
}
return src
}
func SimpleEncryptMap(m maps.Map) (base64String string, err error) {
mJSON, err := json.Marshal(m)
if err != nil {
return "", err
}
data := SimpleEncrypt(mJSON)
return base64.StdEncoding.EncodeToString(data), nil
}
func SimpleDecryptMap(base64String string) (maps.Map, error) {
data, err := base64.StdEncoding.DecodeString(base64String)
if err != nil {
return nil, err
}
mJSON := SimpleDecrypt(data)
var result = maps.Map{}
err = json.Unmarshal(mJSON, &result)
if err != nil {
return nil, err
}
return result, nil
}
type AES256CFBMethod struct {
block cipher.Block
iv []byte
}
func (this *AES256CFBMethod) Init(key, iv []byte) error {
// 判断key是否为32长度
l := len(key)
if l > 32 {
key = key[:32]
} else if l < 32 {
key = append(key, bytes.Repeat([]byte{' '}, 32-l)...)
}
block, err := aes.NewCipher(key)
if err != nil {
return err
}
this.block = block
// 判断iv长度
l2 := len(iv)
if l2 > aes.BlockSize {
iv = iv[:aes.BlockSize]
} else if l2 < aes.BlockSize {
iv = append(iv, bytes.Repeat([]byte{' '}, aes.BlockSize-l2)...)
}
this.iv = iv
return nil
}
func (this *AES256CFBMethod) Encrypt(src []byte) (dst []byte, err error) {
if len(src) == 0 {
return
}
defer func() {
r := recover()
if r != nil {
err = errors.New("encrypt failed")
}
}()
dst = make([]byte, len(src))
encrypter := cipher.NewCFBEncrypter(this.block, this.iv)
encrypter.XORKeyStream(dst, src)
return
}
func (this *AES256CFBMethod) Decrypt(dst []byte) (src []byte, err error) {
if len(dst) == 0 {
return
}
defer func() {
r := recover()
if r != nil {
err = errors.New("decrypt failed")
}
}()
src = make([]byte, len(dst))
decrypter := cipher.NewCFBDecrypter(this.block, this.iv)
decrypter.XORKeyStream(src, dst)
return
}

View File

@@ -0,0 +1,52 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package utils
import (
"github.com/iwind/TeaGo/maps"
"sync"
"testing"
)
func TestSimpleEncrypt(t *testing.T) {
var arr = []string{"Hello", "World", "People"}
for _, s := range arr {
var value = []byte(s)
encoded := SimpleEncrypt(value)
t.Log(encoded, string(encoded))
decoded := SimpleDecrypt(encoded)
t.Log(decoded, string(decoded))
}
}
func TestSimpleEncrypt_Concurrent(t *testing.T) {
wg := sync.WaitGroup{}
var arr = []string{"Hello", "World", "People"}
wg.Add(len(arr))
for _, s := range arr {
go func(s string) {
defer wg.Done()
t.Log(string(SimpleDecrypt(SimpleEncrypt([]byte(s)))))
}(s)
}
wg.Wait()
}
func TestSimpleEncryptMap(t *testing.T) {
var m = maps.Map{
"s": "Hello",
"i": 20,
"b": true,
}
encodedResult, err := SimpleEncryptMap(m)
if err != nil {
t.Fatal(err)
}
t.Log("result:", encodedResult)
decodedResult, err := SimpleDecryptMap(encodedResult)
if err != nil {
t.Fatal(err)
}
t.Log(decodedResult)
}

View File

@@ -12,6 +12,7 @@ type List struct {
itemsMap map[int64]int64 // itemId => timestamp
locker sync.Mutex
ticker *time.Ticker
}
func NewList() *List {
@@ -21,10 +22,7 @@ func NewList() *List {
}
}
func (this *List) Add(itemId int64, expiredAt int64) {
if expiredAt <= time.Now().Unix() {
return
}
func (this *List) Add(itemId int64, expiresAt int64) {
this.locker.Lock()
defer this.locker.Unlock()
@@ -34,17 +32,17 @@ func (this *List) Add(itemId int64, expiredAt int64) {
this.removeItem(itemId)
}
expireItemMap, ok := this.expireMap[expiredAt]
expireItemMap, ok := this.expireMap[expiresAt]
if ok {
expireItemMap[itemId] = true
} else {
expireItemMap = ItemMap{
itemId: true,
}
this.expireMap[expiredAt] = expireItemMap
this.expireMap[expiresAt] = expireItemMap
}
this.itemsMap[itemId] = expiredAt
this.itemsMap[itemId] = expiresAt
}
func (this *List) Remove(itemId int64) {
@@ -64,21 +62,22 @@ func (this *List) GC(timestamp int64, callback func(itemId int64)) {
}
func (this *List) StartGC(callback func(itemId int64)) {
ticker := time.NewTicker(1 * time.Second)
this.ticker = time.NewTicker(1 * time.Second)
lastTimestamp := int64(0)
for range ticker.C {
for range this.ticker.C {
timestamp := time.Now().Unix()
if lastTimestamp == 0 {
lastTimestamp = timestamp - 3600
}
// 防止死循环
if lastTimestamp > timestamp {
continue
}
for i := lastTimestamp; i <= timestamp; i++ {
this.GC(timestamp, callback)
if timestamp >= lastTimestamp {
for i := lastTimestamp; i <= timestamp; i++ {
this.GC(i, callback)
}
} else {
for i := timestamp; i <= lastTimestamp; i++ {
this.GC(i, callback)
}
}
// 这样做是为了防止系统时钟突变

View File

@@ -58,6 +58,10 @@ func TestList_Start_GC(t *testing.T) {
list.Add(2, time.Now().Unix()+1)
list.Add(3, time.Now().Unix()+2)
list.Add(4, time.Now().Unix()+5)
list.Add(5, time.Now().Unix()+5)
list.Add(6, time.Now().Unix()+6)
list.Add(7, time.Now().Unix()+6)
list.Add(8, time.Now().Unix()+6)
go func() {
list.StartGC(func(itemId int64) {
@@ -66,7 +70,7 @@ func TestList_Start_GC(t *testing.T) {
})
}()
time.Sleep(10 * time.Second)
time.Sleep(20 * time.Second)
}
func TestList_ManyItems(t *testing.T) {

View File

@@ -8,7 +8,7 @@ import (
"strings"
)
// 将IP转换为整型
// IP2Long 将IP转换为整型
// 注意IPv6没有顺序
func IP2Long(ip string) uint64 {
if len(ip) == 0 {

View File

@@ -0,0 +1,35 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package jsonutils
import (
"encoding/json"
"github.com/iwind/TeaGo/maps"
)
func MapToObject(m maps.Map, ptr interface{}) error {
if m == nil {
return nil
}
mJSON, err := json.Marshal(m)
if err != nil {
return err
}
return json.Unmarshal(mJSON, ptr)
}
func ObjectToMap(ptr interface{}) (maps.Map, error) {
if ptr == nil {
return maps.Map{}, nil
}
ptrJSON, err := json.Marshal(ptr)
if err != nil {
return nil, err
}
var result = maps.Map{}
err = json.Unmarshal(ptrJSON, &result)
if err != nil {
return nil, err
}
return result, nil
}

View File

@@ -0,0 +1,46 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package jsonutils
import (
"github.com/iwind/TeaGo/assert"
"github.com/iwind/TeaGo/maps"
"testing"
)
func TestMapToObject(t *testing.T) {
a := assert.NewAssertion(t)
type typeA struct {
B int `json:"b"`
C bool `json:"c"`
}
{
var obj = &typeA{B: 1, C: true}
m, err := ObjectToMap(obj)
if err != nil {
t.Fatal(err)
}
PrintT(m, t)
a.IsTrue(m.GetInt("b") == 1)
a.IsTrue(m.GetBool("c") == true)
}
{
var obj = &typeA{}
err := MapToObject(maps.Map{
"b": 1024,
"c": true,
}, obj)
if err != nil {
t.Fatal(err)
}
if obj == nil {
t.Fatal("obj should not be nil")
}
a.IsTrue(obj.B == 1024)
a.IsTrue(obj.C == true)
PrintT(obj, t)
}
}

View File

@@ -0,0 +1,17 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package jsonutils
import (
"encoding/json"
"testing"
)
func PrintT(obj interface{}, t *testing.T) {
data, err := json.MarshalIndent(obj, "", " ")
if err != nil {
t.Log(err)
} else {
t.Log(string(data))
}
}

98
internal/utils/unzip.go Normal file
View File

@@ -0,0 +1,98 @@
package utils
import (
"archive/zip"
"errors"
"io"
"os"
"strings"
)
type Unzip struct {
zipFile string
targetDir string
stripPrefix string
}
func NewUnzip(zipFile string, targetDir string, stripPrefix string) *Unzip {
return &Unzip{
zipFile: zipFile,
targetDir: targetDir,
stripPrefix: stripPrefix,
}
}
func (this *Unzip) Run() error {
if len(this.zipFile) == 0 {
return errors.New("zip file should not be empty")
}
if len(this.targetDir) == 0 {
return errors.New("target dir should not be empty")
}
reader, err := zip.OpenReader(this.zipFile)
if err != nil {
return err
}
defer func() {
_ = reader.Close()
}()
for _, file := range reader.File {
info := file.FileInfo()
filename := file.Name
if len(this.stripPrefix) > 0 {
filename = strings.TrimPrefix(filename, this.stripPrefix)
}
target := this.targetDir + "/" + filename
// 目录
if info.IsDir() {
stat, err := os.Stat(target)
if err != nil {
if !os.IsNotExist(err) {
return err
} else {
err = os.MkdirAll(target, info.Mode())
if err != nil {
return err
}
}
} else if !stat.IsDir() {
err = os.MkdirAll(target, info.Mode())
if err != nil {
return err
}
}
continue
}
// 文件
err := func(file *zip.File, target string) error {
fileReader, err := file.Open()
if err != nil {
return err
}
defer func() {
_ = fileReader.Close()
}()
fileWriter, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, file.FileInfo().Mode())
if err != nil {
return err
}
defer func() {
_ = fileWriter.Close()
}()
_, err = io.Copy(fileWriter, fileReader)
return err
}(file, target)
if err != nil {
return err
}
}
return nil
}

View File

@@ -8,7 +8,23 @@ import (
type AllowAction struct {
}
func (this *AllowAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *AllowAction) Init(waf *WAF) error {
return nil
}
func (this *AllowAction) Code() string {
return ActionAllow
}
func (this *AllowAction) IsAttack() bool {
return false
}
func (this *AllowAction) WillChange() bool {
return false
}
func (this *AllowAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
// do nothing
return true
}

View File

@@ -0,0 +1,21 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package waf
import "net/http"
type BaseAction struct {
}
// CloseConn 关闭连接
func (this *BaseAction) CloseConn(writer http.ResponseWriter) error {
// 断开连接
hijack, ok := writer.(http.Hijacker)
if ok {
conn, _, err := hijack.Hijack()
if err == nil {
return conn.Close()
}
}
return nil
}

View File

@@ -23,12 +23,48 @@ type BlockAction struct {
StatusCode int `yaml:"statusCode" json:"statusCode"`
Body string `yaml:"body" json:"body"` // supports HTML
URL string `yaml:"url" json:"url"`
Timeout int32 `yaml:"timeout" json:"timeout"`
}
func (this *BlockAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *BlockAction) Init(waf *WAF) error {
if waf.DefaultBlockAction != nil {
if this.StatusCode <= 0 {
this.StatusCode = waf.DefaultBlockAction.StatusCode
}
if len(this.Body) == 0 {
this.Body = waf.DefaultBlockAction.Body
}
if len(this.URL) == 0 {
this.URL = waf.DefaultBlockAction.URL
}
if this.Timeout <= 0 {
this.Timeout = waf.DefaultBlockAction.Timeout
}
}
return nil
}
func (this *BlockAction) Code() string {
return ActionBlock
}
func (this *BlockAction) IsAttack() bool {
return true
}
func (this *BlockAction) WillChange() bool {
return true
}
func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
if this.Timeout > 0 {
// 加入到黑名单
SharedIPBlackLIst.Add(IPTypeAll, request.WAFRemoteIP(), time.Now().Unix()+int64(this.Timeout))
}
if writer != nil {
// if status code eq 444, we close the connection
if this.StatusCode == 444 {
// close the connection
defer func() {
hijack, ok := writer.(http.Hijacker)
if ok {
conn, _, _ := hijack.Hijack()
@@ -37,7 +73,7 @@ func (this *BlockAction) Perform(waf *WAF, request *requests.Request, writer htt
return
}
}
}
}()
// output response
if this.StatusCode > 0 {

View File

@@ -1,11 +1,14 @@
package waf
import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
"github.com/iwind/TeaGo/types"
"github.com/iwind/TeaGo/maps"
stringutil "github.com/iwind/TeaGo/utils/string"
"net/http"
"net/url"
"strings"
"time"
)
@@ -13,27 +16,63 @@ var captchaSalt = stringutil.Rand(32)
const (
CaptchaSeconds = 600 // 10 minutes
CaptchaPath = "/WAF/VERIFY/CAPTCHA"
)
type CaptchaAction struct {
Life int32 `yaml:"life" json:"life"`
Language string `yaml:"language" json:"language"` // 语言zh-CN, en-US ...
AddToWhiteList bool `yaml:"addToWhiteList" json:"addToWhiteList"` // 是否加入到白名单
}
func (this *CaptchaAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) {
// TEAWEB_CAPTCHA:
cookie, err := request.Cookie("TEAWEB_WAF_CAPTCHA")
if err == nil && cookie != nil && len(cookie.Value) > 32 {
m := cookie.Value[:32]
timestamp := cookie.Value[32:]
if stringutil.Md5(captchaSalt+timestamp) == m && time.Now().Unix() < types.Int64(timestamp) { // verify md5
return true
func (this *CaptchaAction) Init(waf *WAF) error {
return nil
}
func (this *CaptchaAction) Code() string {
return ActionCaptcha
}
func (this *CaptchaAction) IsAttack() bool {
return false
}
func (this *CaptchaAction) WillChange() bool {
return true
}
func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
// 是否在白名单中
if SharedIPWhiteList.Contains("set:"+set.Id, request.WAFRemoteIP()) {
return true
}
refURL := request.WAFRaw().URL.String()
// 覆盖配置
if strings.HasPrefix(refURL, CaptchaPath) {
info := request.WAFRaw().URL.Query().Get("info")
if len(info) > 0 {
m, err := utils.SimpleDecryptMap(info)
if err == nil && m != nil {
refURL = m.GetString("url")
}
}
}
refURL := request.URL.String()
if len(request.Referer()) > 0 {
refURL = request.Referer()
var captchaConfig = maps.Map{
"action": this,
"timestamp": time.Now().Unix(),
"url": refURL,
"setId": set.Id,
}
http.Redirect(writer, request.Raw(), "/WAFCAPTCHA?url="+url.QueryEscape(refURL), http.StatusTemporaryRedirect)
info, err := utils.SimpleEncryptMap(captchaConfig)
if err != nil {
remotelogs.Error("WAF_CAPTCHA_ACTION", "encode captcha config failed: "+err.Error())
return true
}
http.Redirect(writer, request.WAFRaw(), CaptchaPath+"?info="+url.QueryEscape(info), http.StatusTemporaryRedirect)
return false
}

View File

@@ -0,0 +1,13 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package waf
import "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
type ActionCategory = string
const (
ActionCategoryAllow ActionCategory = firewallconfigs.HTTPFirewallActionCategoryAllow
ActionCategoryBlock ActionCategory = firewallconfigs.HTTPFirewallActionCategoryBlock
ActionCategoryVerify ActionCategory = firewallconfigs.HTTPFirewallActionCategoryVerify
)

View File

@@ -0,0 +1,10 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package waf
import "github.com/iwind/TeaGo/maps"
type ActionConfig struct {
Code string `yaml:"code" json:"code"`
Options maps.Map `yaml:"options" json:"options"`
}

View File

@@ -2,11 +2,12 @@ package waf
import "reflect"
// action definition
// ActionDefinition action definition
type ActionDefinition struct {
Name string
Code ActionString
Description string
Category string // category: block, verify, allow
Instance ActionInterface
Type reflect.Type
}

View File

@@ -0,0 +1,71 @@
package waf
import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
"github.com/iwind/TeaGo/maps"
"net/http"
"net/url"
"time"
)
const (
Get302Path = "/WAF/VERIFY/GET"
)
// Get302Action
// 原理: origin url --> 302 verify url --> origin url
// TODO 将来支持meta refresh验证
type Get302Action struct {
BaseAction
Life int32 `yaml:"life" json:"life"`
}
func (this *Get302Action) Init(waf *WAF) error {
return nil
}
func (this *Get302Action) Code() string {
return ActionGet302
}
func (this *Get302Action) IsAttack() bool {
return false
}
func (this *Get302Action) WillChange() bool {
return true
}
func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
// 仅限于Get
if request.WAFRaw().Method != http.MethodGet {
return true
}
// 是否已经在白名单中
if SharedIPWhiteList.Contains("set:"+set.Id, request.WAFRemoteIP()) {
return true
}
var m = maps.Map{
"url": request.WAFRaw().URL.String(),
"timestamp": time.Now().Unix(),
"life": this.Life,
"setId": set.Id,
}
info, err := utils.SimpleEncryptMap(m)
if err != nil {
remotelogs.Error("WAF_GET_302_ACTION", "encode info failed: "+err.Error())
return true
}
http.Redirect(writer, request.WAFRaw(), Get302Path+"?info="+url.QueryEscape(info), http.StatusFound)
// 关闭连接
_ = this.CloseConn(writer)
return true
}

View File

@@ -10,13 +10,29 @@ type GoGroupAction struct {
GroupId string `yaml:"groupId" json:"groupId"`
}
func (this *GoGroupAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) {
group := waf.FindRuleGroup(this.GroupId)
if group == nil || !group.IsOn {
func (this *GoGroupAction) Init(waf *WAF) error {
return nil
}
func (this *GoGroupAction) Code() string {
return ActionGoGroup
}
func (this *GoGroupAction) IsAttack() bool {
return false
}
func (this *GoGroupAction) WillChange() bool {
return true
}
func (this *GoGroupAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
nextGroup := waf.FindRuleGroup(this.GroupId)
if nextGroup == nil || !nextGroup.IsOn {
return true
}
b, set, err := group.MatchRequest(request)
b, nextSet, err := nextGroup.MatchRequest(request)
if err != nil {
logs.Error(err)
return true
@@ -26,9 +42,5 @@ func (this *GoGroupAction) Perform(waf *WAF, request *requests.Request, writer h
return true
}
actionObject := FindActionInstance(set.Action, set.ActionOptions)
if actionObject == nil {
return true
}
return actionObject.Perform(waf, request, writer)
return nextSet.PerformActions(waf, nextGroup, request, writer)
}

View File

@@ -11,17 +11,33 @@ type GoSetAction struct {
SetId string `yaml:"setId" json:"setId"`
}
func (this *GoSetAction) Perform(waf *WAF, request *requests.Request, writer http.ResponseWriter) (allow bool) {
group := waf.FindRuleGroup(this.GroupId)
if group == nil || !group.IsOn {
func (this *GoSetAction) Init(waf *WAF) error {
return nil
}
func (this *GoSetAction) Code() string {
return ActionGoSet
}
func (this *GoSetAction) IsAttack() bool {
return false
}
func (this *GoSetAction) WillChange() bool {
return true
}
func (this *GoSetAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
nextGroup := waf.FindRuleGroup(this.GroupId)
if nextGroup == nil || !nextGroup.IsOn {
return true
}
set := group.FindRuleSet(this.SetId)
if set == nil || !set.IsOn {
nextSet := nextGroup.FindRuleSet(this.SetId)
if nextSet == nil || !nextSet.IsOn {
return true
}
b, err := set.MatchRequest(request)
b, err := nextSet.MatchRequest(request)
if err != nil {
logs.Error(err)
return true
@@ -29,9 +45,5 @@ func (this *GoSetAction) Perform(waf *WAF, request *requests.Request, writer htt
if !b {
return true
}
actionObject := FindActionInstance(set.Action, set.ActionOptions)
if actionObject == nil {
return true
}
return actionObject.Perform(waf, request, writer)
return nextSet.PerformActions(waf, nextGroup, request, writer)
}

Some files were not shown because too many files have changed in this diff Show More