Compare commits

..

40 Commits

Author SHA1 Message Date
刘祥超
57cb00edf0 修复无法添加IP到本地防火墙的问题 2022-09-04 17:06:19 +08:00
刘祥超
3fb39b479a 优化IP名单锁 2022-09-03 21:10:13 +08:00
刘祥超
4a1daff143 增加简化的缓存条件设置 2022-09-03 18:16:35 +08:00
刘祥超
dd1dbd424e 修复WAF IP名单测试用例的一个可能异常 2022-09-03 09:56:36 +08:00
刘祥超
305cb4b46e 优化WAF中IP名单 2022-09-03 09:54:25 +08:00
刘祥超
ef90dce29b 优化节点进程退出逻辑 2022-09-02 16:12:58 +08:00
刘祥超
3cb69f4c71 将IP加入黑名单时,同时也会关闭此IP相关的连接 2022-09-02 15:20:58 +08:00
刘祥超
af4cd05df2 增加一处注释 2022-09-01 09:06:21 +08:00
刘祥超
64e0ae80b7 DDoS防护增加秒级连接速率限制 2022-08-31 10:00:38 +08:00
刘祥超
8bba228745 优化代码 2022-08-30 18:49:21 +08:00
刘祥超
8cc06e6707 鉴权成功时也在访问日志中记录鉴权类型 2022-08-30 17:28:58 +08:00
刘祥超
52fdee2eeb 修复访问控制后缓存Key包含认证参数的问题 2022-08-30 12:04:01 +08:00
刘祥超
b5f52dd136 优化鉴权 2022-08-30 11:24:23 +08:00
刘祥超
abda886de5 如果系统安装了ntpdate,则自动尝试利用ntpdate同步时间 2022-08-27 15:28:53 +08:00
刘祥超
18f08525b9 WAF和其他请求关闭连接时更加快速 2022-08-27 10:49:16 +08:00
刘祥超
b2a9a31fe5 增加edge-node bandwidth命令查看实时带宽 2022-08-26 16:47:46 +08:00
刘祥超
f578114aeb 修复HTTPS连接无法记录带宽的问题,优化带宽计算方法 2022-08-26 16:47:42 +08:00
刘祥超
bf4f47fc35 DDoS防护增加单IP TCP新连接速率黑名单 2022-08-26 11:31:43 +08:00
刘祥超
0be951742a WAF优化captcha和js_cookie的失败计数器/增强js_cookie的安全性 2022-08-25 17:06:52 +08:00
刘祥超
59d3d6ae4b WAF标签动作匹配之后可以继续尝试匹配别的分组中的规则集 2022-08-25 16:44:44 +08:00
刘祥超
d061876f7e 增加Javascript Cookie验证 2022-08-25 15:35:32 +08:00
刘祥超
ddaec82415 优化RPC获取服务实例方式 2022-08-24 20:04:46 +08:00
刘祥超
8afd00f00d 删除HTTP/2自动关闭Body的逻辑 2022-08-24 16:43:32 +08:00
刘祥超
0b306f0a22 忽略部分HTTP/2错误提示 2022-08-24 16:30:38 +08:00
刘祥超
b36a36172b 优化代码 2022-08-23 14:53:39 +08:00
刘祥超
770278bbbc 实现IP库检查更新 2022-08-23 14:32:39 +08:00
刘祥超
ca24818571 优化缓存索引内存使用 2022-08-22 09:44:09 +08:00
刘祥超
cd0af22655 优化代码 2022-08-22 08:31:39 +08:00
刘祥超
96cb8d8af7 IP库改为手动初始化 2022-08-21 23:09:47 +08:00
刘祥超
91face15bf 使用新版IP库 2022-08-21 20:37:49 +08:00
刘祥超
6f52df63a5 只有系统内存超过3GB的才缓存Hash 2022-08-20 12:09:15 +08:00
刘祥超
509d81dc66 修复检测是否为高速硬盘只能检查sda/vda设备的问题 2022-08-20 11:59:49 +08:00
刘祥超
817f2a6f91 大幅提升缓存索引性能 2022-08-20 11:47:57 +08:00
刘祥超
d74e10c7a8 缓存任务忽略连接API错误 2022-08-20 08:15:16 +08:00
刘祥超
1a1280b76e 优化代码 2022-08-19 14:50:26 +08:00
刘祥超
85d3b1169f 修复Gif中透明图转换WebP失败的问题 2022-08-19 13:39:46 +08:00
刘祥超
1b32292e0c WebP压缩Gif时,自动跳过转换失败的帧,并只提示最后一次错误 2022-08-19 13:27:18 +08:00
刘祥超
e6ba44fb2f [SQLITE]使用事务批量提交一些缓存相关任务 2022-08-17 21:04:00 +08:00
刘祥超
9b6e022f46 将版本修改为0.5.2 2022-08-17 18:56:48 +08:00
刘祥超
4e9ab19fc0 修复因为fsnotify包引用而无法编译的问题 2022-08-17 12:12:40 +08:00
94 changed files with 1864 additions and 2024 deletions

View File

@@ -52,7 +52,6 @@ function build() {
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
# we support TOA on linux/amd64 only
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

View File

@@ -318,6 +318,21 @@ func main() {
}
}
})
app.On("bandwidth", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
var statsMap = maps.NewMap(reply.Params).Get("stats")
statsJSON, err := json.MarshalIndent(statsMap, "", " ")
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
fmt.Println(string(statsJSON))
})
app.Run(func() {
var node = nodes.NewNode()
node.Start()

11
go.mod
View File

@@ -4,8 +4,7 @@ go 1.18
replace (
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
github.com/fsnotify/fsnotify => /Users/WorkSpace/Projects/fsnotify
rogchap.com/v8go => /Users/Workspace/Projects/v8go
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
)
require (
@@ -21,7 +20,7 @@ require (
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5
github.com/klauspost/compress v1.15.8
github.com/mattn/go-sqlite3 v1.14.9
github.com/mdlayher/netlink v1.4.2
@@ -29,9 +28,9 @@ require (
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
github.com/shirou/gopsutil/v3 v3.22.2
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
golang.org/x/text v0.3.7
google.golang.org/grpc v1.45.0
gopkg.in/yaml.v3 v3.0.1
@@ -41,7 +40,7 @@ require (
require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/webp v1.1.0 // indirect
github.com/chai2010/webp v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect

77
go.sum
View File

@@ -1,17 +1,13 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -20,8 +16,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -37,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -47,15 +42,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -87,22 +76,20 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20220807023459-448081424640 h1:nBVQzDI4mrQS+Egg+Li6BGiTToBsv+XTck+BItgI52k=
github.com/iwind/TeaGo v0.0.0-20220807023459-448081424640/go.mod h1:+K2l6Num4Evl0jH7TYlZJ1oFJX8sA8YUC31Pb+I1mJk=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55 h1:shQNx0flJFBwKsGE7Hs3bI2bDz+YF0zl/4qE8B2KRiY=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5 h1:SgKhrqRhWVpu0ZKxQM8MGjdhy7hlH3Xax8snwsZWSic=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -119,7 +106,6 @@ github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -155,19 +141,9 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -182,7 +158,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
@@ -196,41 +171,30 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -259,22 +223,15 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -292,7 +249,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -302,11 +258,10 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -318,7 +273,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
@@ -332,7 +286,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
@@ -341,7 +294,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
@@ -360,22 +312,13 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -96,13 +96,13 @@ func (this *FileList) Reset() error {
}
func (this *FileList) Add(hash string, item *Item) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
err := db.Add(hash, item)
err := db.AddAsync(hash, item)
if err != nil {
return err
}
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
}
func (this *FileList) Exist(hash string) (bool, error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// 如果Hash列表里不存在那么必然不存在
if !db.hashMap.Exist(hash) {
return false, nil
}
var item = this.memoryCache.Read(hash)
if item != nil {
return true, nil
@@ -225,7 +230,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
return err
}
if notFound {
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return db.WrapError(err)
}
@@ -266,7 +271,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
_ = check
row := db.statStmt.QueryRow()
var row = db.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
@@ -291,13 +296,13 @@ func (this *FileList) Count() (int64, error) {
// IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
return db.IncreaseHit(hash)
return db.IncreaseHitAsync(hash)
}
// OnAdd 添加事件
@@ -326,17 +331,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
return fnv.HashString(hash) % CountFileDB
}
func (this *FileList) getDB(hash string) *FileListDB {
func (this *FileList) GetDB(hash string) *FileListDB {
return this.dbList[fnv.HashString(hash)%CountFileDB]
}
func (this *FileList) remove(hash string) (notFound bool, err error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// HashMap中不存在则确定不存在
if !db.hashMap.Exist(hash) {
return true, nil
}
defer db.hashMap.Delete(hash)
// 从缓存中删除
this.memoryCache.Delete(hash)
@@ -357,14 +368,14 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
return false, err
}
_, err = db.deleteByHashStmt.Exec(hash)
err = db.DeleteAsync(hash)
if err != nil {
return false, db.WrapError(err)
}
atomic.AddInt64(&this.total, -1)
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return false, db.WrapError(err)
}

View File

@@ -6,6 +6,8 @@ import (
"database/sql"
"errors"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/iwind/TeaGo/types"
@@ -21,6 +23,10 @@ type FileListDB struct {
readDB *dbs.DB
writeDB *dbs.DB
writeBatch *dbs.Batch
hashMap *FileListHashMap
itemsTableName string
hitsTableName string
@@ -30,31 +36,48 @@ type FileListDB struct {
isReady bool
// cacheItems
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
selectByHashStmt *dbs.Stmt // 使用hash查询数据
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
insertSQL string
selectByHashStmt *dbs.Stmt // 使用hash查询数据
selectHashListStmt *dbs.Stmt
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
deleteByHashSQL string
statStmt *dbs.Stmt // 统计
purgeStmt *dbs.Stmt // 清理
deleteAllStmt *dbs.Stmt // 删除所有数据
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
// hits
insertHitStmt *dbs.Stmt // 写入数据
increaseHitStmt *dbs.Stmt // 增加点击量
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
insertHitSQL string // 写入数据
increaseHitSQL string // 增加点击量
deleteHitByHashSQL string // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
}
func NewFileListDB() *FileListDB {
return &FileListDB{}
return &FileListDB{
hashMap: NewFileListHashMap(),
}
}
func (this *FileListDB) Open(dbPath string) error {
this.dbPath = dbPath
// 动态调整Cache值
var cacheSize = 32000
var memoryGB = utils.SystemMemoryGB()
if memoryGB >= 8 {
cacheSize += 32000 * memoryGB / 8
}
// write db
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize)+"&_secure_delete=FAST")
if err != nil {
return errors.New("open write database failed: " + err.Error())
}
@@ -69,13 +92,22 @@ func (this *FileListDB) Open(dbPath string) error {
}**/
this.writeDB = dbs.NewDB(writeDB)
this.writeBatch = dbs.NewBatch(writeDB, 4)
this.writeBatch.OnFail(func(err error) {
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error())
})
goman.New(func() {
this.writeBatch.Exec()
})
if teaconst.EnableDBStat {
this.writeBatch.EnableStat(true)
this.writeDB.EnableStat(true)
}
// read db
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize))
if err != nil {
return errors.New("open read database failed: " + err.Error())
}
@@ -119,7 +151,8 @@ func (this *FileListDB) Init() error {
return err
}
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
if err != nil {
return err
}
@@ -129,7 +162,10 @@ func (this *FileListDB) Init() error {
return err
}
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
if err != nil {
return err
}
@@ -151,17 +187,11 @@ func (this *FileListDB) Init() error {
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
if err != nil {
return err
}
this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
@@ -170,6 +200,14 @@ func (this *FileListDB) Init() error {
this.isReady = true
// 加载HashMap
go func() {
err := this.hashMap.Load(this)
if err != nil {
remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
}
}()
return nil
}
@@ -181,12 +219,25 @@ func (this *FileListDB) Total() int64 {
return this.total
}
func (this *FileListDB) Add(hash string, item *Item) error {
func (this *FileListDB) AddAsync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
return nil
}
func (this *FileListDB) AddSync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
// 放入队列
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
if err != nil {
return this.WrapError(err)
@@ -195,6 +246,23 @@ func (this *FileListDB) Add(hash string, item *Item) error {
return nil
}
func (this *FileListDB) DeleteAsync(hash string) error {
this.hashMap.Delete(hash)
this.writeBatch.Add(this.deleteByHashSQL, hash)
return nil
}
func (this *FileListDB) DeleteSync(hash string) error {
this.hashMap.Delete(hash)
_, err := this.deleteByHashStmt.Exec(hash)
if err != nil {
return err
}
return nil
}
func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) {
if !this.isReady {
return nil, nil
@@ -250,10 +318,36 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
return
}
func (this *FileListDB) IncreaseHit(hash string) error {
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
rows, err := this.selectHashListStmt.Query(lastId)
if err != nil {
return nil, 0, err
}
var id int64
var hash string
for rows.Next() {
err = rows.Scan(&id, &hash)
if err != nil {
_ = rows.Close()
return
}
maxId = id
hashList = append(hashList, hash)
}
_ = rows.Close()
return
}
func (this *FileListDB) IncreaseHitAsync(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return this.WrapError(err)
this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
return nil
}
func (this *FileListDB) DeleteHitAsync(hash string) error {
this.writeBatch.Add(this.deleteHitByHashSQL, hash)
return nil
}
func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -288,6 +382,8 @@ func (this *FileListDB) CleanAll() error {
return this.WrapError(err)
}
this.hashMap.Clean()
return nil
}
@@ -304,6 +400,9 @@ func (this *FileListDB) Close() error {
if this.selectByHashStmt != nil {
_ = this.selectByHashStmt.Close()
}
if this.selectHashListStmt != nil {
_ = this.selectHashListStmt.Close()
}
if this.deleteByHashStmt != nil {
_ = this.deleteByHashStmt.Close()
}
@@ -319,16 +418,6 @@ func (this *FileListDB) Close() error {
if this.listOlderItemsStmt != nil {
_ = this.listOlderItemsStmt.Close()
}
if this.insertHitStmt != nil {
_ = this.insertHitStmt.Close()
}
if this.increaseHitStmt != nil {
_ = this.increaseHitStmt.Close()
}
if this.deleteHitByHashStmt != nil {
_ = this.deleteHitByHashStmt.Close()
}
if this.lfuHitsStmt != nil {
_ = this.lfuHitsStmt.Close()
}
@@ -349,6 +438,10 @@ func (this *FileListDB) Close() error {
}
}
if this.writeBatch != nil {
this.writeBatch.Close()
}
if len(errStrings) == 0 {
return nil
}

View File

@@ -0,0 +1,114 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"math/big"
"sync"
)
// FileListHashMap 文件Hash列表
type FileListHashMap struct {
m map[uint64]zero.Zero
locker sync.RWMutex
isAvailable bool
isReady bool
}
func NewFileListHashMap() *FileListHashMap {
return &FileListHashMap{
m: map[uint64]zero.Zero{},
isAvailable: false,
isReady: false,
}
}
func (this *FileListHashMap) Load(db *FileListDB) error {
// 如果系统内存过小,我们不缓存
if utils.SystemMemoryGB() < 3 {
return nil
}
this.isAvailable = true
var lastId int64
for {
hashList, maxId, err := db.ListHashes(lastId)
if err != nil {
return err
}
if len(hashList) == 0 {
break
}
this.AddHashes(hashList)
lastId = maxId
}
this.isReady = true
return nil
}
func (this *FileListHashMap) Add(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
this.m[this.bigInt(hash)] = zero.New()
this.locker.Unlock()
}
func (this *FileListHashMap) AddHashes(hashes []string) {
if !this.isAvailable {
return
}
this.locker.Lock()
for _, hash := range hashes {
this.m[this.bigInt(hash)] = zero.New()
}
this.locker.Unlock()
}
func (this *FileListHashMap) Delete(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
delete(this.m, this.bigInt(hash))
this.locker.Unlock()
}
func (this *FileListHashMap) Exist(hash string) bool {
if !this.isAvailable {
return true
}
if !this.isReady {
// 只有完全Ready时才能判断是否为false
return true
}
this.locker.RLock()
_, ok := this.m[this.bigInt(hash)]
this.locker.RUnlock()
return ok
}
func (this *FileListHashMap) Clean() {
this.locker.Lock()
this.m = map[uint64]zero.Zero{}
this.locker.Unlock()
}
func (this *FileListHashMap) IsReady() bool {
return this.isReady
}
func (this *FileListHashMap) bigInt(hash string) uint64 {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
return bigInt.Uint64()
}

View File

@@ -0,0 +1,91 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"math/big"
"runtime"
"strconv"
"testing"
)
func TestFileListHashMap_Memory(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = caches.NewFileListHashMap()
for i := 0; i < 1_000_000; i++ {
m.Add(stringutil.Md5(types.String(i)))
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_Memory2(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = map[uint64]zero.Zero{}
for i := 0; i < 1_000_000; i++ {
m[uint64(i)] = zero.New()
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_BigInt(t *testing.T) {
for _, s := range []string{"1", "2", "3", "123", "123456"} {
var hash = stringutil.Md5(s)
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
}
}
func TestFileListHashMap_Load(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
var m = caches.NewFileListHashMap()
err = m.Load(list.GetDB("abc"))
if err != nil {
t.Fatal(err)
}
m.Add("abc")
for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
t.Log(hash, "=>", m.Exist(hash))
}
}
func Benchmark_BigInt(b *testing.B) {
var hash = stringutil.Md5("123456")
b.ResetTimer()
for i := 0; i < b.N; i++ {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
_ = bigInt.Uint64()
}
}

View File

@@ -69,8 +69,8 @@ func TestFileList_Add_Many(t *testing.T) {
_ = list.Close()
}()
before := time.Now()
for i := 0; i < 100_000; i++ {
var before = time.Now()
for i := 0; i < 10_000_000; i++ {
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &caches.Item{
Key: u,
@@ -290,12 +290,12 @@ func TestFileList_Stat(t *testing.T) {
}
func TestFileList_Count(t *testing.T) {
list := caches.NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
var before = time.Now()
count, err := list.Count()
if err != nil {
t.Fatal(err)
@@ -367,6 +367,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
}

View File

@@ -6,7 +6,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"strconv"
"sync"
@@ -16,7 +15,7 @@ var SharedManager = NewManager()
func init() {
events.On(events.EventQuit, func() {
logs.Println("CACHE", "quiting cache manager")
remotelogs.Println("CACHE", "quiting cache manager")
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
})
}

117
internal/conns/map.go Normal file
View File

@@ -0,0 +1,117 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package conns
import (
"net"
"sync"
)
var SharedMap = NewMap()
type Map struct {
m map[string]map[int]net.Conn // ip => { port => Conn }
locker sync.RWMutex
}
func NewMap() *Map {
return &Map{
m: map[string]map[int]net.Conn{},
}
}
func (this *Map) Add(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
this.m[ip] = map[int]net.Conn{
port: conn,
}
} else {
connMap[port] = conn
}
}
func (this *Map) Remove(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
return
}
delete(connMap, port)
if len(connMap) == 0 {
delete(this.m, ip)
}
}
func (this *Map) CountIPConns(ip string) int {
this.locker.RLock()
var l = len(this.m[ip])
this.locker.RUnlock()
return l
}
func (this *Map) CloseIPConns(ip string) {
var conns = []net.Conn{}
this.locker.RLock()
connMap, ok := this.m[ip]
// 复制防止在Close时产生并发冲突
if ok {
for _, conn := range connMap {
conns = append(conns, conn)
}
}
// 需要在Close之前结束防止死循环
this.locker.RUnlock()
if ok {
for _, conn := range conns {
_ = conn.Close()
}
// 这里不需要从 m 中删除,因为关闭时会自然触发回调
}
}
func (this *Map) AllConns() []net.Conn {
this.locker.RLock()
defer this.locker.RUnlock()
var result = []net.Conn{}
for _, m := range this.m {
for _, conn := range m {
result = append(result, conn)
}
}
return result
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.5.1"
Version = "0.5.2"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -195,14 +195,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// new connections rate
var newConnectionsRate = tcpConfig.NewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = 100000
// new connections rate (minutely)
var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = 100000
}
}
var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
if newConnectionsMinutelyRateBlockTimeout < 0 {
newConnectionsMinutelyRateBlockTimeout = 0
}
// new connections rate (secondly)
var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = 10000
}
}
var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
if newConnectionsSecondlyRateBlockTimeout < 0 {
newConnectionsSecondlyRateBlockTimeout = 0
}
// 检查是否有变化
var hasChanges = false
@@ -215,7 +232,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
hasChanges = true
break
}
@@ -251,6 +272,7 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// TODO 让用户选择是drop还是reject
if maxConnectionsPerIP > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
var stderr = &bytes.Buffer{}
@@ -261,14 +283,47 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
if newConnectionsRate > 0 {
// TODO 思考是否有惩罚机制
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
// 超过一定速率就drop或者加入黑名单分钟
// TODO 让用户选择是drop还是reject
if newConnectionsMinutelyRate > 0 {
if newConnectionsMinutelyRateBlockTimeout > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
} else {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
// 超过一定速率就drop或者加入黑名单
// TODO 让用户选择是drop还是reject
if newConnectionsSecondlyRate > 0 {
if newConnectionsSecondlyRateBlockTimeout > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
} else {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
}
@@ -335,14 +390,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
for _, rule := range rules {
var pieces = this.decodeUserData(rule.UserData())
if len(pieces) != 4 {
if len(pieces) < 4 {
continue
}
if pieces[0] != "tcp" {
continue
}
switch pieces[2] {
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
err := chain.DeleteRule(rule)
if err != nil {
return err

View File

@@ -0,0 +1,47 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package firewalls
import (
"github.com/iwind/TeaGo/types"
"strings"
"sync"
"time"
)
type BaseFirewall struct {
locker sync.Mutex
latestIPTimes []string // [ip@time, ....]
}
// 检查是否在最近添加过
func (this *BaseFirewall) checkLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -4,6 +4,7 @@ package firewalls
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
@@ -11,15 +12,22 @@ import (
"strings"
)
type firewalldCmd struct {
cmd *exec.Cmd
denyIP string
}
type Firewalld struct {
BaseFirewall
isReady bool
exe string
cmdQueue chan *exec.Cmd
cmdQueue chan *firewalldCmd
}
func NewFirewalld() *Firewalld {
var firewalld = &Firewalld{
cmdQueue: make(chan *exec.Cmd, 4096),
cmdQueue: make(chan *firewalldCmd, 4096),
}
path, err := exec.LookPath("firewall-cmd")
@@ -41,13 +49,19 @@ func NewFirewalld() *Firewalld {
func (this *Firewalld) init() {
goman.New(func() {
for cmd := range this.cmdQueue {
for c := range this.cmdQueue {
var cmd = c.cmd
err := cmd.Run()
if err != nil {
if strings.HasPrefix(err.Error(), "Warning:") {
continue
}
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
} else {
// 关闭连接
if len(c.denyIP) > 0 {
conns.SharedMap.CloseIPConns(c.denyIP)
}
}
}
})
@@ -72,7 +86,7 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
return nil
}
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
return nil
}
@@ -82,12 +96,12 @@ func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol
{
var cmd = exec.Command(this.exe, "--add-port="+port, "--permanent")
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
{
var cmd = exec.Command(this.exe, "--add-port="+port)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
}
@@ -99,7 +113,7 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
return nil
}
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
return nil
}
@@ -108,12 +122,12 @@ func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol str
{
var cmd = exec.Command(this.exe, "--remove-port="+port, "--permanent")
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
{
var cmd = exec.Command(this.exe, "--remove-port="+port)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
return nil
@@ -131,6 +145,12 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -140,7 +160,7 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
this.pushCmd(cmd, ip)
return nil
}
@@ -148,6 +168,12 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) e
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -158,10 +184,13 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) e
}
var cmd = exec.Command(this.exe, args...)
if async {
this.pushCmd(cmd)
this.pushCmd(cmd, ip)
return nil
}
// 关闭连接
defer conns.SharedMap.CloseIPConns(ip)
err := cmd.Run()
if err != nil {
return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
@@ -173,6 +202,7 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -180,14 +210,14 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
for _, action := range []string{"reject", "drop"} {
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
return nil
}
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
func (this *Firewalld) pushCmd(cmd *exec.Cmd, denyIP string) {
select {
case this.cmdQueue <- cmd:
case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
default:
// we discard the command
}

View File

@@ -7,6 +7,7 @@ package firewalls
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
@@ -100,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
}
type NFTablesFirewall struct {
BaseFirewall
conn *nftables.Conn
isReady bool
version string
@@ -344,6 +347,14 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
return errors.New("invalid ip '" + ip + "'")
}
// 尝试关闭连接
conns.SharedMap.CloseIPConns(ip)
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
if async {
select {
case this.dropIPQueue <- &blockIPItem{
@@ -357,6 +368,9 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
return nil
}
// 再次尝试关闭连接
defer conns.SharedMap.CloseIPConns(ip)
if strings.Contains(ip, ":") { // ipv6
if this.denyIPv6Set == nil {
return errors.New("ipv6 ip set is nil")
@@ -433,3 +447,35 @@ func (this *NFTablesFirewall) readVersion(nftPath string) string {
}
return versionMatches[1]
}
// 检查是否在最近添加过
func (this *NFTablesFirewall) existLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -1,130 +0,0 @@
// 源码改自https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
package iplibrary
import (
"errors"
"os"
"strconv"
"strings"
)
const (
IndexBlockLength = 12
)
var err error
type IP2Region struct {
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
lastIndexPtr int64
totalBlocks int64
dbData []byte
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
func getIpInfo(cityId int64, line []byte) *IpInfo {
lineSlice := strings.Split(string(line), "|")
ipInfo := &IpInfo{}
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
ipInfo.Country = lineSlice[0]
ipInfo.Region = lineSlice[1]
ipInfo.Province = lineSlice[2]
ipInfo.City = lineSlice[3]
ipInfo.ISP = lineSlice[4]
return ipInfo
}
func NewIP2Region(path string) (*IP2Region, error) {
var region = &IP2Region{}
region.dbData, err = os.ReadFile(path)
if err != nil {
return nil, err
}
region.firstIndexPtr = region.ipLongAtOffset(0)
region.lastIndexPtr = region.ipLongAtOffset(4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
return region, nil
}
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
ip, err := ip2long(ipStr)
if err != nil {
return nil, err
}
h := this.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m*IndexBlockLength
sip := this.ipLongAtOffset(p)
if ip < sip {
h = m - 1
} else {
eip := this.ipLongAtOffset(p + 4)
if ip > eip {
l = m + 1
} else {
dataPtr = this.ipLongAtOffset(p + 8)
break
}
}
}
if dataPtr == 0 {
return nil, nil
}
dataLen := (dataPtr >> 24) & 0xFF
dataPtr = dataPtr & 0x00FFFFFF
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
}
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
return int64(this.dbData[offset]) |
int64(this.dbData[offset+1])<<8 |
int64(this.dbData[offset+2])<<16 |
int64(this.dbData[offset+3])<<24
}
func ip2long(IpStr string) (int64, error) {
bits := strings.Split(IpStr, ".")
if len(bits) != 4 {
return 0, errors.New("ip format error")
}
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24-8*i)
}
return sum, nil
}

View File

@@ -18,7 +18,8 @@ import (
type IPListDB struct {
db *sql.DB
itemTableName string
itemTableName string
deleteExpiredItemsStmt *sql.Stmt
deleteItemStmt *sql.Stmt
insertItemStmt *sql.Stmt
@@ -53,7 +54,11 @@ func (this *IPListDB) init() error {
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = this.dir + "/ip_list.db"
_ = os.Remove(path + "-shm")
_ = os.Remove(path + "-wal")
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
@@ -194,12 +199,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
return 0
}
row := this.selectMaxVersionStmt.QueryRow()
var row = this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err = row.Scan(&version)
err := row.Scan(&version)
if err != nil {
return 0
}

View File

@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.AddItem(&pb.IPItem{
Id: 1,
IpFrom: "192.168.1.101",
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.Close()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -1,13 +0,0 @@
package iplibrary
type LibraryInterface interface {
// Load 加载数据库文件
Load(dbPath string) error
// Lookup 查询IP
// 返回结果有可能为空
Lookup(ip string) (*Result, error)
// Close 关闭数据库文件
Close()
}

View File

@@ -1,83 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"net"
"strings"
)
type IP2RegionLibrary struct {
db *IP2Region
}
func (this *IP2RegionLibrary) Load(dbPath string) error {
db, err := NewIP2Region(dbPath)
if err != nil {
return err
}
this.db = db
return nil
}
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
// 暂不支持IPv6
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
if this.db == nil {
return nil, errors.New("library has not been loaded")
}
defer func() {
// 防止panic发生
err := recover()
if err != nil {
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
}
}()
info, err := this.db.MemorySearch(ip)
if err != nil {
return nil, err
}
if info == nil {
return nil, nil
}
if info.Country == "0" {
info.Country = ""
}
if info.Region == "0" {
info.Region = ""
}
if info.Province == "0" {
info.Province = ""
}
if info.City == "0" {
info.City = ""
}
if info.ISP == "0" {
info.ISP = ""
}
return &Result{
CityId: info.CityId,
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
ISP: info.ISP,
}, nil
}
func (this *IP2RegionLibrary) Close() {
}

View File

@@ -1,114 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
"sync"
"testing"
"time"
)
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
var mem = &runtime.MemStats{}
runtime.ReadMemStats(mem)
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var mem2 = &runtime.MemStats{}
runtime.ReadMemStats(mem2)
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
}
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"8.8.9.9"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var count = 4000
var wg = sync.WaitGroup{}
wg.Add(count)
for i := 0; i < count; i++ {
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
}()
}
wg.Done()
t.Log("ok")
}
func TestIP2RegionLibrary_Memory(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 1_000_000; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
var library = &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = library.Lookup("8.8.8.8")
}
}

View File

@@ -1,95 +0,0 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/files"
"github.com/iwind/TeaGo/types"
"regexp"
"strings"
)
var SharedManager = NewManager()
var SharedLibrary LibraryInterface
func init() {
events.On(events.EventLoaded, func() {
// 初始化
library, err := SharedManager.Load()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
return
}
SharedLibrary = library
})
}
type Manager struct {
code string
}
func NewManager() *Manager {
return &Manager{}
}
func (this *Manager) Load() (LibraryInterface, error) {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return nil, err
}
config := nodeConfig.GlobalConfig
if config == nil {
config = &serverconfigs.GlobalConfig{}
}
// 当前正在使用的IP库代号
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
dir := Tea.Root + "/resources/ipdata/" + code
var lastVersion int64 = -1
lastFilename := ""
for _, file := range files.NewFile(dir).List() {
filename := file.Name()
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
if reg.MatchString(filename) { // 先查找有版本号的
result := reg.FindStringSubmatch(filename)
version := types.Int64(result[1])
if version > lastVersion {
lastVersion = version
lastFilename = filename
}
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
if lastVersion == -1 {
lastFilename = filename
lastVersion = 0
}
}
}
if len(lastFilename) == 0 {
return nil, errors.New("ip library file not found")
}
var libraryPtr LibraryInterface
switch code {
case serverconfigs.IPLibraryTypeIP2Region:
libraryPtr = &IP2RegionLibrary{}
default:
return nil, errors.New("invalid ip library code '" + code + "'")
}
err = libraryPtr.Load(dir + "/" + lastFilename)
if err != nil {
return nil, err
}
return libraryPtr, nil
}

View File

@@ -1,155 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/types"
"os"
"sync"
"time"
)
var SharedCityManager = NewCityManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCityManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCityManager.Stop()
})
}
// CityManager 中国省份信息管理
type CityManager struct {
ticker *time.Ticker
cacheFile string
cityMap map[string]int64 // provinceName_cityName => cityName
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCityManager() *CityManager {
return &CityManager{
cacheFile: Tea.Root + "/configs/region_city.json.cache",
cityMap: map[string]int64{},
}
}
func (this *CityManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("City_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
}
}
func (this *CityManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
this.locker.RLock()
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *CityManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.cityMap = m
}
return nil
}
// 更新城市信息
func (this *CityManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCityRPC().FindAllRegionCities(rpcClient.Context(), &pb.FindAllRegionCitiesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, city := range resp.RegionCities {
for _, code := range city.Codes {
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.cityMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,14 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewCityManager(t *testing.T) {
var manager = NewCityManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup(16, "许昌市"))
}

View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"os"
"sync"
"time"
)
var SharedCountryManager = NewCountryManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCountryManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCountryManager.Stop()
})
}
// CountryManager 国家/地区信息管理
type CountryManager struct {
ticker *time.Ticker
cacheFile string
countryMap map[string]int64 // countryName => countryId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCountryManager() *CountryManager {
return &CountryManager{
cacheFile: Tea.Root + "/configs/region_country.json.cache",
countryMap: map[string]int64{},
}
}
func (this *CountryManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
}
}
func (this *CountryManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
this.locker.RLock()
countryId, _ = this.countryMap[countryName]
this.locker.RUnlock()
return countryId
}
// 从缓存中读取
func (this *CountryManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.countryMap = m
}
return nil
}
// 更新国家信息
func (this *CountryManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCountryRPC().FindAllRegionCountries(rpcClient.Context(), &pb.FindAllRegionCountriesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, country := range resp.RegionCountries {
for _, code := range country.Codes {
m[code] = country.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.countryMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestCountryManager_load(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop(t *testing.T) {
manager := NewCountryManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop_skip(t *testing.T) {
manager := NewCountryManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestCountryManager_Lookup(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
}
func BenchmarkCountryManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewCountryManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("中国")
}
}

View File

@@ -43,7 +43,7 @@ type IPListManager struct {
func NewIPListManager() *IPListManager {
return &IPListManager{
pageSize: 500,
pageSize: 1000,
listMap: map[int64]*IPList{},
}
}
@@ -111,15 +111,19 @@ func (this *IPListManager) init() {
var size int64 = 1000
for {
items, err := db.ReadItems(offset, size)
var l = len(items)
if err != nil {
remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
} else {
if len(items) == 0 {
if l == 0 {
break
}
this.processItems(items, false)
if int64(l) < size {
break
}
}
offset += int64(len(items))
offset += int64(l)
}
}
}
@@ -144,7 +148,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
if err != nil {
return false, err
}
itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
Version: this.version,
Size: this.pageSize,
})
@@ -155,7 +159,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
}
return false, err
}
items := itemsResp.IpItems
var items = itemsResp.IpItems
if len(items) == 0 {
return false, nil
}
@@ -183,7 +187,6 @@ func (this *IPListManager) FindList(listId int64) *IPList {
}
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
this.locker.Lock()
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
var list *IPList
@@ -203,11 +206,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
list = GlobalWhiteIPList
}
} else { // 其他List
this.locker.Lock()
list = this.listMap[item.ListId]
this.locker.Unlock()
}
if list == nil {
list = NewIPList()
this.locker.Lock()
this.listMap[item.ListId] = list
this.locker.Unlock()
}
changedLists[list] = zero.New()
@@ -246,8 +253,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
changedList.Sort()
}
this.locker.Unlock()
if fromRemote {
var latestVersion = items[len(items)-1].Version
if latestVersion > this.version {

View File

@@ -1,154 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"os"
"sync"
"time"
)
var SharedProviderManager = NewProviderManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProviderManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProviderManager.Stop()
})
}
// ProviderManager 中国省份信息管理
type ProviderManager struct {
ticker *time.Ticker
cacheFile string
providerMap map[string]int64 // name => id
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProviderManager() *ProviderManager {
return &ProviderManager{
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
providerMap: map[string]int64{},
}
}
func (this *ProviderManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
}
}
func (this *ProviderManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
this.locker.RLock()
providerId, _ = this.providerMap[providerName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *ProviderManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.providerMap = m
}
return nil
}
// 更新服务商信息
func (this *ProviderManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProviderRPC().FindAllRegionProviders(rpcClient.Context(), &pb.FindAllRegionProvidersRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, provider := range resp.RegionProviders {
for _, code := range provider.Codes {
m[code] = provider.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.providerMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,15 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewProviderManager(t *testing.T) {
var manager = NewProviderManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("阿里云"))
t.Log(manager.Lookup("阿里云2"))
}

View File

@@ -1,160 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"os"
"sync"
"time"
)
const (
ChinaCountryId int64 = 1
)
var SharedProvinceManager = NewProvinceManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProvinceManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProvinceManager.Stop()
})
}
// ProvinceManager 中国省份信息管理
type ProvinceManager struct {
ticker *time.Ticker
cacheFile string
provinceMap map[string]int64 // provinceName => provinceId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProvinceManager() *ProvinceManager {
return &ProvinceManager{
cacheFile: Tea.Root + "/configs/region_province.json.cache",
provinceMap: map[string]int64{},
}
}
func (this *ProvinceManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
}
}
func (this *ProvinceManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
this.locker.RLock()
provinceId, _ = this.provinceMap[provinceName]
this.locker.RUnlock()
return provinceId
}
// 从缓存中读取
func (this *ProvinceManager) load() error {
data, err := os.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.provinceMap = m
}
return nil
}
// 更新省份信息
func (this *ProvinceManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProvinceRPC().FindAllRegionProvincesWithRegionCountryId(rpcClient.Context(), &pb.FindAllRegionProvincesWithRegionCountryIdRequest{
RegionCountryId: ChinaCountryId,
})
if err != nil {
return err
}
m := map[string]int64{}
for _, province := range resp.RegionProvinces {
for _, code := range province.Codes {
m[code] = province.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.provinceMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = os.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestProvinceManager_load(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop(t *testing.T) {
manager := NewProvinceManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop_skip(t *testing.T) {
manager := NewProvinceManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestProvinceManager_Lookup(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
}
func BenchmarkProvinceManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewProvinceManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("安徽省")
}
}

View File

@@ -1,26 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestManager_Load(t *testing.T) {
dbs.NotifyReady()
manager := NewManager()
lib, err := manager.Load()
if err != nil {
t.Fatal(err)
}
t.Log(lib.Lookup("1.2.3.4"))
t.Log(lib.Lookup("2.3.4.5"))
t.Log(lib.Lookup("200.200.200.200"))
t.Log(lib.Lookup("202.106.0.20"))
}
func TestNewManager(t *testing.T) {
dbs.NotifyReady()
t.Log(SharedLibrary)
}

View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"os"
"time"
)
var SharedUpdater = NewUpdater()
func init() {
events.On(events.EventStart, func() {
goman.New(func() {
SharedUpdater.Start()
})
})
events.On(events.EventQuit, func() {
SharedUpdater.Stop()
})
}
// Updater IP库更新程序
type Updater struct {
ticker *time.Ticker
}
// NewUpdater 获取新对象
func NewUpdater() *Updater {
return &Updater{}
}
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新因为通常不需要更新IP库
this.ticker = time.NewTicker(1 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
}
}
func (this *Updater) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
// 单次任务
func (this *Updater) loop() error {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return err
}
if nodeConfig.GlobalConfig == nil {
return nil
}
code := nodeConfig.GlobalConfig.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
if err != nil {
return err
}
lib := libraryResp.IpLibrary
if lib == nil || lib.File == nil {
return nil
}
typeInfo := serverconfigs.FindIPLibraryWithType(code)
if typeInfo == nil {
return errors.New("invalid ip library code '" + code + "'")
}
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
// 是否已经存在
_, err = os.Stat(path)
if err == nil {
return nil
}
// 开始下载
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
if err != nil {
return err
}
chunkIds := fileChunkIdsResp.FileChunkIds
if len(chunkIds) == 0 {
return nil
}
isOk := false
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer func() {
// 如果保存不成功就直接删除
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
for _, chunkId := range chunkIds {
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
chunk := chunkResp.FileChunk
if chunk == nil {
continue
}
_, err = fp.Write(chunk.Data)
if err != nil {
return err
}
}
err = fp.Close()
if err != nil {
return err
}
// 重新加载
library, err := SharedManager.Load()
if err != nil {
return err
}
SharedLibrary = library
isOk = true
return nil
}

View File

@@ -1,18 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestUpdater_loop(t *testing.T) {
dbs.NotifyReady()
updater := NewUpdater()
err := updater.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -4,6 +4,7 @@ package metrics
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"strconv"
"sync"
@@ -11,7 +12,15 @@ import (
var SharedManager = NewManager()
func init() {
events.On(events.EventQuit, func() {
SharedManager.Quit()
})
}
type Manager struct {
isQuiting bool
tasks map[int64]*Task // itemId => *Task
categoryTasks map[string][]*Task // category => []*Task
locker sync.RWMutex
@@ -29,6 +38,10 @@ func NewManager() *Manager {
}
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
if this.isQuiting {
return
}
this.locker.Lock()
defer this.locker.Unlock()
@@ -101,6 +114,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
// Add 添加数据
func (this *Manager) Add(obj MetricInterface) {
if this.isQuiting {
return
}
this.locker.RLock()
for _, task := range this.categoryTasks[obj.MetricCategory()] {
task.Add(obj)
@@ -119,3 +136,17 @@ func (this *Manager) HasTCPMetrics() bool {
func (this *Manager) HasUDPMetrics() bool {
return this.hasUDPMetrics
}
// Quit 退出管理器
func (this *Manager) Quit() {
this.isQuiting = true
remotelogs.Println("METRIC_MANAGER", "quit")
this.locker.Lock()
for _, task := range this.tasks {
_ = task.Stop()
}
this.tasks = map[int64]*Task{}
this.locker.Unlock()
}

View File

@@ -16,6 +16,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
@@ -89,7 +90,11 @@ func (this *Task) Init() error {
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
_ = os.Remove(path + "-shm")
_ = os.Remove(path + "-wal")
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
@@ -424,7 +429,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
return nil, err
}
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
MetricStats: pbStats,
Time: currentTime,
ServerId: serverId,

View File

@@ -69,7 +69,7 @@ func (this *ValueQueue) Loop() error {
}
for value := range this.valuesChan {
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
Item: value.Item,
ValueJSON: value.ValueJSON,
CreatedAt: value.CreatedAt,

View File

@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
if err != nil {
statusErr, ok:= status.FromError(err)
statusErr, ok := status.FromError(err)
if ok {
logs.Println(statusErr.Code())
}

View File

@@ -73,7 +73,7 @@ func (this *APIStream) loop() error {
cancelFunc()
}()
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
nodeStream, err := rpcClient.NodeRPC.NodeStream(ctx)
if err != nil {
if this.isQuiting {
return nil

View File

@@ -5,12 +5,14 @@ package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
"net"
"os"
@@ -22,6 +24,8 @@ import (
// ClientConn 客户端连接
type ClientConn struct {
BaseClientConn
once sync.Once
isTLS bool
@@ -31,29 +35,28 @@ type ClientConn struct {
isLO bool // 是否为环路
hasResetSYNFlood bool
BaseClientConn
}
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
if quickClose {
// TCP
tcpConn, ok := conn.(*net.TCPConn)
if ok {
// TODO 可以在配置中设置此值
_ = tcpConn.SetLinger(nodeconfigs.DefaultTCPLinger)
}
}
func NewClientConn(rawConn net.Conn, isTLS bool, quickClose bool) net.Conn {
// 是否为环路
var remoteAddr = conn.RemoteAddr().String()
var remoteAddr = rawConn.RemoteAddr().String()
var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
return &ClientConn{
BaseClientConn: BaseClientConn{rawConn: conn},
var conn = &ClientConn{
BaseClientConn: BaseClientConn{rawConn: rawConn},
isTLS: isTLS,
isLO: isLO,
}
if quickClose {
// TODO 可以在配置中设置此值
_ = conn.SetLinger(nodeconfigs.DefaultTCPLinger)
}
// 加入到Map
conns.SharedMap.Add(conn)
return conn
}
func (this *ClientConn) Read(b []byte) (n int, err error) {
@@ -110,11 +113,10 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
func (this *ClientConn) Write(b []byte) (n int, err error) {
n, err = this.rawConn.Write(b)
if n > 0 {
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
// 统计当前服务带宽
if this.serverId > 0 {
if !this.isLO { // 环路不统计带宽,避免缓存预热等行为产生带宽
if !this.isLO || Tea.IsTesting() { // 环路不统计带宽,避免缓存预热等行为产生带宽
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
}
}
@@ -132,6 +134,9 @@ func (this *ClientConn) Close() error {
// 不能加条件限制,因为服务配置随时有变化
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
// 从conn map中移除
conns.SharedMap.Remove(this)
return err
}
@@ -177,6 +182,11 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
if timeout <= 0 {
timeout = 600
}
// 关闭当前连接
_ = this.SetLinger(0)
_ = this.Close()
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击当前1分钟"+types.String(result)+"次空连接")
}
}

View File

@@ -2,7 +2,10 @@
package nodes
import "net"
import (
"crypto/tls"
"net"
)
type BaseClientConn struct {
rawConn net.Conn
@@ -42,6 +45,17 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
// SetServerId 设置服务ID
func (this *BaseClientConn) SetServerId(serverId int64) {
this.serverId = serverId
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
nativeConn, ok := conn.NetConn().(ClientConnInterface)
if ok {
nativeConn.SetServerId(serverId)
}
case *ClientConn:
conn.SetServerId(serverId)
}
}
// ServerId 读取当前连接绑定的服务ID
@@ -52,6 +66,17 @@ func (this *BaseClientConn) ServerId() int64 {
// SetUserId 设置所属服务的用户ID
func (this *BaseClientConn) SetUserId(userId int64) {
this.userId = userId
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
nativeConn, ok := conn.NetConn().(ClientConnInterface)
if ok {
nativeConn.SetUserId(userId)
}
case *ClientConn:
conn.SetUserId(userId)
}
}
// UserId 获取当前连接所属服务的用户ID
@@ -66,9 +91,20 @@ func (this *BaseClientConn) RawIP() string {
}
// TCPConn 转换为TCPConn
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
conn, ok := this.rawConn.(*net.TCPConn)
return conn, ok
func (this *BaseClientConn) TCPConn() (tcpConn *net.TCPConn, ok bool) {
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
var internalConn = conn.NetConn()
clientConn, ok := internalConn.(*ClientConn)
if ok {
return clientConn.TCPConn()
}
tcpConn, ok = internalConn.(*net.TCPConn)
default:
tcpConn, ok = this.rawConn.(*net.TCPConn)
}
return
}
// SetLinger 设置Linger

View File

@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"net"
"time"
)
// ClientListener 客户端网络监听
@@ -42,10 +43,28 @@ func (this *ClientListener) Accept() (net.Conn, error) {
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err == nil {
canGoNext, _ := iplibrary.AllowIP(ip, 0)
var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
expiresAt, ok := waf.SharedIPBlackList.ContainsExpires(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if ok {
var timeout = expiresAt - time.Now().Unix()
if timeout > 0 {
canGoNext = false
if !canGoNext || beingDenied {
if timeout > 3600 {
timeout = 3600
}
// 使用本地防火墙延长封禁
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
// 这里 int(int64) 转换的前提是限制了 timeout <= 3600否则将有整型溢出的风险
_ = fw.DropSourceIP(ip, int(timeout), true)
}
}
}
}
if !canGoNext {
tcpConn, ok := conn.(*net.TCPConn)
if ok {
_ = tcpConn.SetLinger(0)
@@ -53,14 +72,6 @@ func (this *ClientListener) Accept() (net.Conn, error) {
_ = conn.Close()
// 使用本地防火墙延长封禁
if beingDenied {
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
_ = fw.DropSourceIP(ip, 120, true)
}
}
return this.Accept()
}
}

View File

@@ -96,7 +96,7 @@ Loop:
this.rpcClient = client
}
_, err := this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
_, err := this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
if err != nil {
// 是否包含了invalid UTF-8
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
@@ -105,7 +105,7 @@ Loop:
}
// 重新提交
_, err = this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
return err
}

View File

@@ -55,7 +55,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
// logs.PrintAsJSON(accessLog)
//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
accessLog,
}})
if err != nil {
@@ -99,7 +99,7 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
accessLog,
}})
if err != nil {

View File

@@ -81,7 +81,7 @@ func (this *HTTPCacheTaskManager) Start() {
if rpcClient != nil {
for taskReq := range this.taskQueue {
_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
_, err := rpcClient.ServerRPC.PurgeServerCache(rpcClient.Context(), taskReq)
if err != nil {
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
}
@@ -104,8 +104,12 @@ func (this *HTTPCacheTaskManager) Loop() error {
return err
}
resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
resp, err := rpcClient.HTTPCacheTaskKeyRPC.FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
if err != nil {
// 忽略连接错误
if rpc.IsConnError(err) {
return nil
}
return err
}
@@ -131,7 +135,7 @@ func (this *HTTPCacheTaskManager) Loop() error {
pbResults = append(pbResults, pbResult)
}
_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
_, err = rpcClient.HTTPCacheTaskKeyRPC.UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
if err != nil {
return err
}

View File

@@ -6,10 +6,10 @@ import (
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -677,7 +677,7 @@ func (this *HTTPRequest) Format(source string) string {
case "remoteAddrValue":
return this.requestRemoteAddr(false)
case "rawRemoteAddr":
addr := this.RawReq.RemoteAddr
var addr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
@@ -928,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
// geo
if prefix == "geo" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "country.name":
if result != nil {
return result.Country
if result != nil && result.IsOk() {
return result.CountryName()
}
return ""
case "country.id":
if result != nil {
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
if result != nil && result.IsOk() {
return types.String(result.CountryId())
}
return "0"
case "province.name":
if result != nil {
return result.Province
if result != nil && result.IsOk() {
return result.ProvinceName()
}
return ""
case "province.id":
if result != nil {
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
if result != nil && result.IsOk() {
return types.String(result.ProvinceId())
}
return "0"
case "city.name":
if result != nil {
return result.City
if result != nil && result.IsOk() {
return result.CityName()
}
return ""
case "city.id":
if result != nil {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 {
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
} else {
return "0"
}
if result != nil && result.IsOk() {
return types.String(result.CityId())
}
return "0"
case "town.name":
if result != nil && result.IsOk() {
return result.TownName()
}
return ""
case "town.id":
if result != nil && result.IsOk() {
return types.String(result.TownId())
}
return "0"
}
@@ -971,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
// ips
if prefix == "isp" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "name":
if result != nil {
return result.ISP
if result != nil && result.IsOk() {
return result.ProviderName()
}
case "id":
if result != nil {
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
if result != nil && result.IsOk() {
return types.String(result.ProviderId())
}
return "0"
}
@@ -1098,7 +1103,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
}
// Remote-Addr
remoteAddr := this.RawReq.RemoteAddr
var remoteAddr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
if supportVar {
@@ -1167,7 +1172,7 @@ func (this *HTTPRequest) requestRemoteUser() string {
// Path 请求的URL中路径部分
func (this *HTTPRequest) Path() string {
uri, err := url.ParseRequestURI(this.rawURI)
uri, err := url.ParseRequestURI(this.uri)
if err != nil {
return ""
}
@@ -1315,7 +1320,7 @@ func (this *HTTPRequest) RemoteAddr() string {
}
func (this *HTTPRequest) RawRemoteAddr() string {
addr := this.RawReq.RemoteAddr
var addr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
@@ -1423,11 +1428,16 @@ func (this *HTTPRequest) Done() {
func (this *HTTPRequest) Close() {
this.Done()
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
if requestConn == nil {
return
}
lingerConn, ok := requestConn.(LingerConn)
if ok {
_ = lingerConn.SetLinger(0)
}
conn, ok := requestConn.(net.Conn)
if ok {
_ = conn.Close()
@@ -1721,7 +1731,7 @@ func (this *HTTPRequest) canIgnore(err error) bool {
}
// HTTP/2流错误
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
if err.Error() == "http2: stream closed" || strings.Contains(err.Error(), "stream error") || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
return true
}

View File

@@ -21,7 +21,7 @@ func (this *HTTPRequest) doACME() (shouldStop bool) {
return false
}
keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
keyResp, err := rpcClient.ACMEAuthenticationRPC.FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
if err != nil {
remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
return false

View File

@@ -19,7 +19,10 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
continue
}
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
if !ref.AuthPolicy.MatchRequest(this.RawReq) {
continue
}
ok, newURI, uriChanged, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
subReq.TLS = this.RawReq.TLS
subReq.RemoteAddr = this.RawReq.RemoteAddr
subReq.Host = this.RawReq.Host
@@ -36,24 +39,32 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
return
}
if b {
if ok {
if uriChanged {
this.uri = newURI
}
this.tags = append(this.tags, ref.AuthPolicy.Type)
return
} else {
// Basic Auth比较特殊
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.ReqHost
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
if ok {
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.ReqHost
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
this.writer.WriteHeader(http.StatusUnauthorized)
this.tags = append(this.tags, ref.AuthPolicy.Type)
return true
}
}

View File

@@ -44,12 +44,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
// 检查服务独立的缓存条件
refType := ""
for _, cacheRef := range this.web.Cache.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
if !cacheRef.IsOn {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
if cacheRef.IsReverse {
return
}
@@ -61,12 +60,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
// 检查策略默认的缓存条件
for _, cacheRef := range cachePolicy.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
if !cacheRef.IsOn {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
if cacheRef.IsReverse {
return
}

View File

@@ -34,17 +34,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
}
return this.RawReq.ContentLength + hl, true
case "${countConnection}":
metricNewConnMapLocker.Lock()
_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
if ok {
delete(metricNewConnMap, this.RawReq.RemoteAddr)
}
metricNewConnMapLocker.Unlock()
if ok {
return 1, true
} else {
return 0, false
}
return 1, true
}
return 0, false
}

View File

@@ -252,12 +252,6 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
return
}
// 在HTTP/2下需要防止因为requestBody而导致Content-Length为空的问题
if this.RawReq.ProtoMajor == 2 && this.RawReq.ContentLength == 0 && this.RawReq.Body != nil {
_ = this.RawReq.Body.Close()
this.RawReq.Body = nil
}
// 开始请求
resp, err := client.Do(this.RawReq)
if err != nil {

View File

@@ -2,6 +2,7 @@ package nodes
import (
"bytes"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -161,56 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// 检查地区封禁
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
if iplibrary.SharedLibrary != nil {
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
if err != nil {
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
} else if result != nil {
// 检查国家级别封禁
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
var result = iplib.LookupIP(remoteAddr)
if result != nil && result.IsOk() {
// 检查国家/地区级别封禁
var countryId = result.CountryId()
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
// 检查省份封禁
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
return true, false
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 检查省份封禁
var provinceId = result.ProvinceId()
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
return true, false
}
}
}
@@ -379,6 +372,8 @@ func (this *HTTPRequest) WAFServerId() int64 {
// WAFClose 关闭连接
func (this *HTTPRequest) WAFClose() {
this.Close()
// 这里不要强关IP所有连接避免因为单个服务而影响所有
}
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {

View File

@@ -1007,30 +1007,35 @@ func (this *HTTPWriter) finishWebP() {
Exact: true,
})
} else if gifImage != nil {
anim := gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
anim.WebPAnimEncoderOptions.SetKmin(9)
anim.WebPAnimEncoderOptions.SetKmax(17)
defer anim.ReleaseMemory()
webpConfig := gowebp.NewWebpConfig()
var webpConfig = gowebp.NewWebpConfig()
//webpConfig.SetLossless(1)
webpConfig.SetQuality(f)
var timeline = 0
var lastErr error
for i, img := range gifImage.Image {
err = anim.AddFrame(img, timeline, webpConfig)
if err != nil {
break
// 有错误直接跳过
lastErr = err
err = nil
}
timeline += gifImage.Delay[i] * 10
}
if err == nil {
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
if lastErr != nil {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
}
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
anim.ReleaseMemory()
}
if err != nil && !this.req.canIgnore(err) {

View File

@@ -0,0 +1,103 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
"io"
"os"
)
type IPLibraryUpdater struct {
}
func NewIPLibraryUpdater() *IPLibraryUpdater {
return &IPLibraryUpdater{}
}
// DataDir 文件目录
func (this *IPLibraryUpdater) DataDir() string {
// data/
var dir = Tea.Root + "/data"
stat, err := os.Stat(dir)
if err == nil && stat.IsDir() {
return dir
}
err = os.Mkdir(dir, 0666)
if err == nil {
return dir
}
remotelogs.Error("IP_LIBRARY_UPDATER", "create directory '"+dir+"' failed: "+err.Error())
// 如果不能创建 data/ 目录,那么使用临时目录
return os.TempDir()
}
// FindLatestFile 检查最新的IP库文件
func (this *IPLibraryUpdater) FindLatestFile() (code string, fileId int64, err error) {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return "", 0, err
}
resp, err := rpcClient.IPLibraryArtifactRPC.FindPublicIPLibraryArtifact(rpcClient.Context(), &pb.FindPublicIPLibraryArtifactRequest{})
if err != nil {
return "", 0, err
}
var artifact = resp.IpLibraryArtifact
if artifact == nil {
return
}
return artifact.Code, artifact.FileId, nil
}
// DownloadFile 下载文件
func (this *IPLibraryUpdater) DownloadFile(fileId int64, writer io.Writer) error {
if fileId <= 0 {
return errors.New("invalid fileId: " + types.String(fileId))
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
chunkIdsResp, err := rpcClient.FileChunkRPC.FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: fileId})
if err != nil {
return err
}
for _, chunkId := range chunkIdsResp.FileChunkIds {
chunkResp, err := rpcClient.FileChunkRPC.DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
var chunk = chunkResp.FileChunk
if chunk == nil {
return errors.New("can not find file chunk with chunk id '" + types.String(chunkId) + "'")
}
_, err = writer.Write(chunk.Data)
if err != nil {
return err
}
}
return nil
}
// LogInfo 普通日志
func (this *IPLibraryUpdater) LogInfo(message string) {
remotelogs.Println("IP_LIBRARY_UPDATER", message)
}
// LogError 错误日志
func (this *IPLibraryUpdater) LogError(err error) {
if err == nil {
return
}
remotelogs.Error("IP_LIBRARY_UPDATER", err.Error())
}

View File

@@ -5,7 +5,6 @@ import (
"crypto/tls"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"golang.org/x/net/http2"
"io"
@@ -13,14 +12,11 @@ import (
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
var httpErrorLogger = log.New(io.Discard, "", 0)
var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool
var metricNewConnMapLocker = &sync.Mutex{}
type contextKey struct {
key string
@@ -55,23 +51,10 @@ func (this *HTTPListener) Serve() error {
switch state {
case http.StateNew:
atomic.AddInt64(&this.countActiveConnections, 1)
// 为指标存储连接信息
if sharedNodeConfig.HasHTTPConnectionMetrics() {
metricNewConnMapLocker.Lock()
metricNewConnMap[conn.RemoteAddr().String()] = zero.New()
metricNewConnMapLocker.Unlock()
}
case http.StateActive, http.StateIdle, http.StateHijacked:
// Nothing to do
case http.StateClosed:
atomic.AddInt64(&this.countActiveConnections, -1)
// 移除指标存储连接信息
// 因为中途配置可能有改变,所以暂时不添加条件
metricNewConnMapLocker.Lock()
delete(metricNewConnMap, conn.RemoteAddr().String())
metricNewConnMapLocker.Unlock()
}
},
ConnContext: func(ctx context.Context, conn net.Conn) context.Context {

View File

@@ -4,12 +4,14 @@ import (
"bytes"
"encoding/json"
"errors"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/configs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
@@ -21,6 +23,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/andybalholm/brotli"
"github.com/iwind/TeaGo/Tea"
@@ -78,7 +81,7 @@ func (this *Node) Test() error {
if err != nil {
return errors.New("test rpc failed: " + err.Error())
}
_, err = rpcClient.APINodeRPC().FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
_, err = rpcClient.APINodeRPC.FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
if err != nil {
return errors.New("test rpc failed: " + err.Error())
}
@@ -101,9 +104,6 @@ func (this *Node) Start() {
// 监听signal
this.listenSignals()
// 启动事件
events.Notify(events.EventStart)
// 本地Sock
err := this.listenSock()
if err != nil {
@@ -111,9 +111,19 @@ func (this *Node) Start() {
return
}
// 启动IP库
remotelogs.Println("NODE", "initializing ip library ...")
err = iplib.InitDefault()
if err != nil {
remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
}
// 检查硬盘类型
this.checkDisk()
// 启动事件
events.Notify(events.EventStart)
// 读取API配置
remotelogs.Println("NODE", "init config ...")
err = this.syncConfig(0)
@@ -146,11 +156,21 @@ func (this *Node) Start() {
// 启动同步计时器
this.startSyncTimer()
// 状态变更计时器
// 更新IP库
goman.New(func() {
iplib.NewUpdater(NewIPLibraryUpdater(), 10*time.Minute).Start()
})
// 监控节点运行状态
goman.New(func() {
NewNodeStatusExecutor().Listen()
})
// 同步时间
goman.New(func() {
clock.Start()
})
// 读取配置
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
@@ -294,7 +314,7 @@ func (this *Node) loop() error {
}
var nodeCtx = rpcClient.Context()
tasksResp, err := rpcClient.NodeTaskRPC().FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
tasksResp, err := rpcClient.NodeTaskRPC.FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
if err != nil {
if rpc.IsConnError(err) && !Tea.IsTesting() {
return nil
@@ -312,7 +332,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -331,13 +351,13 @@ func (this *Node) loop() error {
err = this.syncConfig(task.Version)
}
if err != nil {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: false,
Error: err.Error(),
})
} else {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -359,7 +379,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -368,7 +388,7 @@ func (this *Node) loop() error {
return err
}
case "nodeLevelChanged":
levelInfoResp, err := rpcClient.NodeRPC().FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
levelInfoResp, err := rpcClient.NodeRPC.FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
if err != nil {
return err
}
@@ -385,7 +405,7 @@ func (this *Node) loop() error {
sharedNodeConfig.ParentNodes = parentNodes
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -394,7 +414,7 @@ func (this *Node) loop() error {
return err
}
case "ddosProtectionChanged":
resp, err := rpcClient.NodeRPC().FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
resp, err := rpcClient.NodeRPC.FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
if err != nil {
return err
}
@@ -421,7 +441,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -466,7 +486,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
nodeCtx := rpcClient.Context()
// TODO 这里考虑只同步版本号有变更的
configResp, err := rpcClient.NodeRPC().FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
configResp, err := rpcClient.NodeRPC.FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
Version: -1, // 更新所有版本
Compress: true,
NodeTaskVersion: taskVersion,
@@ -557,7 +577,7 @@ func (this *Node) syncServerConfig(serverId int64) error {
if err != nil {
return err
}
resp, err := rpcClient.ServerRPC().ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
resp, err := rpcClient.ServerRPC.ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
if err != nil {
return err
}
@@ -639,7 +659,7 @@ func (this *Node) checkClusterConfig() error {
}
logs.Println("[NODE]registering node to cluster ...")
resp, err := rpcClient.NodeRPC().RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
resp, err := rpcClient.NodeRPC.RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
if err != nil {
return err
}
@@ -778,13 +798,16 @@ func (this *Node) listenSock() error {
},
})
case "conns":
ipConns, serverConns := sharedClientConnLimiter.Conns()
var addrs = []string{}
var connMap = conns.SharedMap.AllConns()
for _, conn := range connMap {
addrs = append(addrs, conn.RemoteAddr().String())
}
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"ipConns": ipConns,
"serverConns": serverConns,
"total": sharedListenerManager.TotalActiveConnections(),
"addrs": addrs,
"total": len(addrs),
},
})
case "dropIP":
@@ -856,6 +879,11 @@ func (this *Node) listenSock() error {
} else {
_ = cmd.ReplyOk()
}
case "bandwidth":
var m = stats.SharedBandwidthStatManager.Map()
_ = cmd.Reply(&gosock.Command{Params: maps.Map{
"stats": m,
}})
}
})
@@ -866,7 +894,7 @@ func (this *Node) listenSock() error {
})
events.OnKey(events.EventQuit, this, func() {
logs.Println("NODE", "quit unix sock")
remotelogs.Println("NODE", "quit unix sock")
_ = this.sock.Close()
})
@@ -1010,10 +1038,13 @@ func (this *Node) reloadServer() {
}
func (this *Node) checkDisk() {
if runtime.GOOS == "linux" {
if runtime.GOOS != "linux" {
return
}
for n := 'a'; n <= 'z'; n++ {
for _, path := range []string{
"/sys/block/vda/queue/rotational",
"/sys/block/sda/queue/rotational",
"/sys/block/vd" + string(n) + "/queue/rotational",
"/sys/block/sd" + string(n) + "/queue/rotational",
} {
data, err := os.ReadFile(path)
if err != nil {
@@ -1022,7 +1053,7 @@ func (this *Node) checkDisk() {
if string(data) == "0" {
teaconst.DiskIsFast = true
}
break
return
}
}
}

View File

@@ -125,7 +125,7 @@ func (this *NodeStatusExecutor) update() {
remotelogs.Error("NODE_STATUS", "failed to open rpc: "+err.Error())
return
}
_, err = rpcClient.NodeRPC().UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
_, err = rpcClient.NodeRPC.UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
StatusJSON: jsonData,
})
if err != nil {

View File

@@ -60,7 +60,7 @@ func (this *OCSPUpdateTask) Loop() error {
return err
}
resp, err := rpcClient.SSLCertRPC().ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
resp, err := rpcClient.SSLCertRPC.ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
Version: this.version,
Size: 100,
})

View File

@@ -81,7 +81,7 @@ func (this *SyncAPINodesTask) Loop() error {
if err != nil {
return err
}
resp, err := rpcClient.APINodeRPC().FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
resp, err := rpcClient.APINodeRPC.FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
if err != nil {
return err
}

View File

@@ -126,7 +126,7 @@ func (this *UpgradeManager) install() error {
var sum = ""
var filename = ""
for {
resp, err := client.NodeRPC().DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
resp, err := client.NodeRPC.DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
Os: runtime.GOOS,
Arch: runtime.GOARCH,
ChunkOffset: offset,

View File

@@ -173,7 +173,6 @@ func ServerSuccess(serverId int64, tag string, description string, logType nodec
}
}
// ServerLog 打印服务相关日志信息
func ServerLog(serverId int64, tag string, description string, logType nodeconfigs.NodeLogType, params maps.Map) {
logs.Println("[" + tag + "]" + description)
@@ -253,6 +252,6 @@ Loop:
return nil
}
_, err = rpcClient.NodeLogRPC().CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
_, err = rpcClient.NodeLogRPC.CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
return err
}

View File

@@ -28,6 +28,27 @@ type RPCClient struct {
conns []*grpc.ClientConn
locker sync.RWMutex
NodeRPC pb.NodeServiceClient
NodeLogRPC pb.NodeLogServiceClient
NodeTaskRPC pb.NodeTaskServiceClient
NodeValueRPC pb.NodeValueServiceClient
HTTPAccessLogRPC pb.HTTPAccessLogServiceClient
HTTPCacheTaskKeyRPC pb.HTTPCacheTaskKeyServiceClient
APINodeRPC pb.APINodeServiceClient
IPLibraryArtifactRPC pb.IPLibraryArtifactServiceClient
IPListRPC pb.IPListServiceClient
IPItemRPC pb.IPItemServiceClient
FileRPC pb.FileServiceClient
FileChunkRPC pb.FileChunkServiceClient
ACMEAuthenticationRPC pb.ACMEAuthenticationServiceClient
ServerRPC pb.ServerServiceClient
ServerDailyStatRPC pb.ServerDailyStatServiceClient
ServerBandwidthStatRPC pb.ServerBandwidthStatServiceClient
MetricStatRPC pb.MetricStatServiceClient
FirewallRPC pb.FirewallServiceClient
SSLCertRPC pb.SSLCertServiceClient
ScriptRPC pb.ScriptServiceClient
}
func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
@@ -35,10 +56,32 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
return nil, errors.New("api config should not be nil")
}
client := &RPCClient{
var client = &RPCClient{
apiConfig: apiConfig,
}
// 初始化RPC实例
client.NodeRPC = pb.NewNodeServiceClient(client)
client.NodeLogRPC = pb.NewNodeLogServiceClient(client)
client.NodeTaskRPC = pb.NewNodeTaskServiceClient(client)
client.NodeValueRPC = pb.NewNodeValueServiceClient(client)
client.HTTPAccessLogRPC = pb.NewHTTPAccessLogServiceClient(client)
client.HTTPCacheTaskKeyRPC = pb.NewHTTPCacheTaskKeyServiceClient(client)
client.APINodeRPC = pb.NewAPINodeServiceClient(client)
client.IPLibraryArtifactRPC = pb.NewIPLibraryArtifactServiceClient(client)
client.IPListRPC = pb.NewIPListServiceClient(client)
client.IPItemRPC = pb.NewIPItemServiceClient(client)
client.FileRPC = pb.NewFileServiceClient(client)
client.FileChunkRPC = pb.NewFileChunkServiceClient(client)
client.ACMEAuthenticationRPC = pb.NewACMEAuthenticationServiceClient(client)
client.ServerRPC = pb.NewServerServiceClient(client)
client.ServerDailyStatRPC = pb.NewServerDailyStatServiceClient(client)
client.ServerBandwidthStatRPC = pb.NewServerBandwidthStatServiceClient(client)
client.MetricStatRPC = pb.NewMetricStatServiceClient(client)
client.FirewallRPC = pb.NewFirewallServiceClient(client)
client.SSLCertRPC = pb.NewSSLCertServiceClient(client)
client.ScriptRPC = pb.NewScriptServiceClient(client)
err := client.init()
if err != nil {
return nil, err
@@ -47,102 +90,10 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
return client, nil
}
func (this *RPCClient) NodeRPC() pb.NodeServiceClient {
return pb.NewNodeServiceClient(this.pickConn())
}
func (this *RPCClient) NodeLogRPC() pb.NodeLogServiceClient {
return pb.NewNodeLogServiceClient(this.pickConn())
}
func (this *RPCClient) NodeTaskRPC() pb.NodeTaskServiceClient {
return pb.NewNodeTaskServiceClient(this.pickConn())
}
func (this *RPCClient) NodeValueRPC() pb.NodeValueServiceClient {
return pb.NewNodeValueServiceClient(this.pickConn())
}
func (this *RPCClient) HTTPAccessLogRPC() pb.HTTPAccessLogServiceClient {
return pb.NewHTTPAccessLogServiceClient(this.pickConn())
}
func (this *RPCClient) HTTPCacheTaskKeyRPC() pb.HTTPCacheTaskKeyServiceClient {
return pb.NewHTTPCacheTaskKeyServiceClient(this.pickConn())
}
func (this *RPCClient) APINodeRPC() pb.APINodeServiceClient {
return pb.NewAPINodeServiceClient(this.pickConn())
}
func (this *RPCClient) IPLibraryRPC() pb.IPLibraryServiceClient {
return pb.NewIPLibraryServiceClient(this.pickConn())
}
func (this *RPCClient) RegionCountryRPC() pb.RegionCountryServiceClient {
return pb.NewRegionCountryServiceClient(this.pickConn())
}
func (this *RPCClient) RegionProvinceRPC() pb.RegionProvinceServiceClient {
return pb.NewRegionProvinceServiceClient(this.pickConn())
}
func (this *RPCClient) RegionCityRPC() pb.RegionCityServiceClient {
return pb.NewRegionCityServiceClient(this.pickConn())
}
func (this *RPCClient) RegionProviderRPC() pb.RegionProviderServiceClient {
return pb.NewRegionProviderServiceClient(this.pickConn())
}
func (this *RPCClient) IPListRPC() pb.IPListServiceClient {
return pb.NewIPListServiceClient(this.pickConn())
}
func (this *RPCClient) IPItemRPC() pb.IPItemServiceClient {
return pb.NewIPItemServiceClient(this.pickConn())
}
func (this *RPCClient) FileRPC() pb.FileServiceClient {
return pb.NewFileServiceClient(this.pickConn())
}
func (this *RPCClient) FileChunkRPC() pb.FileChunkServiceClient {
return pb.NewFileChunkServiceClient(this.pickConn())
}
func (this *RPCClient) ACMEAuthenticationRPC() pb.ACMEAuthenticationServiceClient {
return pb.NewACMEAuthenticationServiceClient(this.pickConn())
}
func (this *RPCClient) ServerRPC() pb.ServerServiceClient {
return pb.NewServerServiceClient(this.pickConn())
}
func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient {
return pb.NewServerDailyStatServiceClient(this.pickConn())
}
func (this *RPCClient) ServerBandwidthStatRPC() pb.ServerBandwidthStatServiceClient {
return pb.NewServerBandwidthStatServiceClient(this.pickConn())
}
func (this *RPCClient) MetricStatRPC() pb.MetricStatServiceClient {
return pb.NewMetricStatServiceClient(this.pickConn())
}
func (this *RPCClient) FirewallRPC() pb.FirewallServiceClient {
return pb.NewFirewallServiceClient(this.pickConn())
}
func (this *RPCClient) SSLCertRPC() pb.SSLCertServiceClient {
return pb.NewSSLCertServiceClient(this.pickConn())
}
// Context 节点上下文信息
func (this *RPCClient) Context() context.Context {
ctx := context.Background()
m := maps.Map{
var ctx = context.Background()
var m = maps.Map{
"timestamp": time.Now().Unix(),
"type": "node",
"userId": 0,
@@ -157,7 +108,7 @@ func (this *RPCClient) Context() context.Context {
utils.PrintError(err)
return context.Background()
}
token := base64.StdEncoding.EncodeToString(data)
var token = base64.StdEncoding.EncodeToString(data)
ctx = metadata.AppendToOutgoingContext(ctx, "nodeId", this.apiConfig.NodeId, "token", token)
return ctx
}
@@ -209,7 +160,7 @@ func (this *RPCClient) UpdateConfig(config *configs.APIConfig) error {
// 初始化
func (this *RPCClient) init() error {
// 重新连接
conns := []*grpc.ClientConn{}
var conns = []*grpc.ClientConn{}
for _, endpoint := range this.apiConfig.RPC.Endpoints {
u, err := url.Parse(endpoint)
if err != nil {
@@ -248,20 +199,24 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
// 检查连接状态
if len(this.conns) > 0 {
availableConns := []*grpc.ClientConn{}
for _, state := range []connectivity.State{connectivity.Ready, connectivity.Idle, connectivity.Connecting} {
var availableConns = []*grpc.ClientConn{}
for _, stateArray := range [][2]connectivity.State{
{connectivity.Ready, connectivity.Idle}, // 优先Ready和Idle
{connectivity.Connecting, connectivity.Connecting},
} {
for _, conn := range this.conns {
if conn.GetState() == state {
var state = conn.GetState()
if state == stateArray[0] || state == stateArray[1] {
availableConns = append(availableConns, conn)
}
}
if len(availableConns) > 0 {
break
return this.randConn(availableConns)
}
}
if len(availableConns) > 0 {
return availableConns[rands.Int(0, len(availableConns)-1)]
return this.randConn(availableConns)
}
// 关闭
@@ -281,5 +236,31 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
return nil
}
return this.conns[rands.Int(0, len(this.conns)-1)]
return this.randConn(this.conns)
}
func (this *RPCClient) Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...grpc.CallOption) error {
var conn = this.pickConn()
if conn == nil {
return errors.New("can not get available grpc connection")
}
return conn.Invoke(ctx, method, args, reply, opts...)
}
func (this *RPCClient) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
var conn = this.pickConn()
if conn == nil {
return nil, errors.New("can not get available grpc connection")
}
return conn.NewStream(ctx, desc, method, opts...)
}
func (this *RPCClient) randConn(conns []*grpc.ClientConn) *grpc.ClientConn {
var l = len(conns)
if l == 0 {
return nil
}
if l == 1 {
return conns[0]
}
return conns[rands.Int(0, l-1)]
}

View File

@@ -17,6 +17,8 @@ import (
var SharedBandwidthStatManager = NewBandwidthStatManager()
const bandwidthTimestampDelim = 2 // N秒平均更为精确
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
@@ -83,7 +85,7 @@ func (this *BandwidthStatManager) Loop() error {
ServerId: stat.ServerId,
Day: stat.Day,
TimeAt: stat.TimeAt,
Bytes: stat.MaxBytes,
Bytes: stat.MaxBytes / bandwidthTimestampDelim,
})
delete(this.m, key)
}
@@ -96,7 +98,7 @@ func (this *BandwidthStatManager) Loop() error {
if err != nil {
return err
}
_, err = rpcClient.ServerBandwidthStatRPC().UploadServerBandwidthStats(rpcClient.Context(), &pb.UploadServerBandwidthStatsRequest{ServerBandwidthStats: pbStats})
_, err = rpcClient.ServerBandwidthStatRPC.UploadServerBandwidthStats(rpcClient.Context(), &pb.UploadServerBandwidthStatsRequest{ServerBandwidthStats: pbStats})
if err != nil {
return err
}
@@ -112,11 +114,18 @@ func (this *BandwidthStatManager) Add(userId int64, serverId int64, bytes int64)
}
var now = time.Now()
var timestamp = now.Unix()
var timestamp = now.Unix() / bandwidthTimestampDelim * bandwidthTimestampDelim // 将时间戳均分成N等份
var day = timeutil.Format("Ymd", now)
var timeAt = timeutil.FormatTime("Hi", now.Unix()/300*300)
var key = types.String(serverId) + "@" + day + "@" + timeAt
// 增加TCP Header尺寸这里默认MTU为1500且默认为IPv4
const mtu = 1500
const tcpHeaderSize = 20
if bytes > mtu {
bytes += bytes * tcpHeaderSize / mtu
}
this.locker.Lock()
stat, ok := this.m[key]
if ok {
@@ -150,3 +159,15 @@ func (this *BandwidthStatManager) Inspect() {
logs.PrintAsJSON(this.m)
this.locker.Unlock()
}
func (this *BandwidthStatManager) Map() map[int64]int64 /** serverId => max bytes **/ {
this.locker.Lock()
defer this.locker.Unlock()
var m = map[int64]int64{}
for _, v := range this.m {
m[v.ServerId] = v.MaxBytes / bandwidthTimestampDelim
}
return m
}

View File

@@ -14,7 +14,7 @@ func TestBandwidthStatManager_Add(t *testing.T) {
manager.Add(1, 1, 10)
manager.Add(1, 1, 10)
time.Sleep(1 * time.Second)
manager.Add(1, 1, 15)
manager.Add(1, 1, 85)
time.Sleep(1 * time.Second)
manager.Add(1, 1, 25)
manager.Add(1, 1, 75)

View File

@@ -1,11 +1,11 @@
package stats
import (
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/monitor"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
@@ -190,31 +190,26 @@ Loop:
var serverId = pieces[0]
var ip = pieces[1]
if iplibrary.SharedLibrary != nil {
result, err := iplibrary.SharedLibrary.Lookup(ip)
if err == nil && result != nil {
if len(result.Country) == 0 {
continue
}
var result = iplib.LookupIP(ip)
if result != nil && result.IsOk() {
var key = serverId + "@" + result.CountryName() + "@" + result.ProvinceName() + "@" + result.CityName()
stat, ok := this.cityMap[key]
if !ok {
stat = &StatItem{}
this.cityMap[key] = stat
}
stat.Bytes += types.Int64(pieces[2])
stat.CountRequests++
if types.Int8(pieces[3]) == 1 {
stat.AttackBytes += types.Int64(pieces[2])
stat.CountAttackRequests++
}
var key = serverId + "@" + result.Country + "@" + result.Province + "@" + result.City
stat, ok := this.cityMap[key]
if !ok {
stat = &StatItem{}
this.cityMap[key] = stat
}
stat.Bytes += types.Int64(pieces[2])
stat.CountRequests++
if types.Int8(pieces[3]) == 1 {
stat.AttackBytes += types.Int64(pieces[2])
stat.CountAttackRequests++
}
if len(result.ISP) > 0 {
this.providerMap[serverId+"@"+result.ISP]++
}
if len(result.ProviderName()) > 0 {
this.providerMap[serverId+"@"+result.ProviderName()]++
}
}
case userAgentString := <-this.userAgentChan:
var atIndex = strings.Index(userAgentString, "@")
if atIndex < 0 {
@@ -329,7 +324,7 @@ func (this *HTTPRequestStatManager) Upload() error {
this.dailyFirewallRuleGroupMap = map[string]int64{}
// 上传数据
_, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
_, err = rpcClient.ServerRPC.UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
Month: timeutil.Format("Ym"),
Day: timeutil.Format("Ymd"),
RegionCities: pbCities,
@@ -351,7 +346,7 @@ func (this *HTTPRequestStatManager) Upload() error {
}
// 再次尝试
_, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
_, err = rpcClient.ServerRPC.UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
Month: timeutil.Format("Ym"),
Day: timeutil.Format("Ymd"),
RegionCities: pbCities,

View File

@@ -1,19 +1,19 @@
package stats
import (
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"testing"
)
func TestHTTPRequestStatManager_Loop_Region(t *testing.T) {
library, err := iplibrary.SharedManager.Load()
err := iplib.InitDefault()
if err != nil {
t.Fatal(err)
}
iplibrary.SharedLibrary = library
manager := NewHTTPRequestStatManager()
var manager = NewHTTPRequestStatManager()
manager.AddRemoteAddr(11, "202.196.0.20", 0, false)
manager.AddRemoteAddr(11, "202.196.0.20", 0, false) // 重复添加一个测试相加
manager.AddRemoteAddr(11, "8.8.8.8", 0, false)
@@ -36,19 +36,13 @@ func TestHTTPRequestStatManager_Loop_Region(t *testing.T) {
}
func TestHTTPRequestStatManager_Loop_UserAgent(t *testing.T) {
library, err := iplibrary.SharedManager.Load()
if err != nil {
t.Fatal(err)
}
iplibrary.SharedLibrary = library
manager := NewHTTPRequestStatManager()
var manager = NewHTTPRequestStatManager()
manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76 Safari/537.36")
manager.AddUserAgent(1, "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
manager.AddUserAgent(1, "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
err = manager.Loop()
err := manager.Loop()
if err != nil {
t.Fatal(err)
}

View File

@@ -210,7 +210,7 @@ func (this *TrafficStatManager) Upload() error {
})
}
_, err = client.ServerDailyStatRPC().UploadServerDailyStats(client.Context(), &pb.UploadServerDailyStatsRequest{
_, err = client.ServerDailyStatRPC.UploadServerDailyStats(client.Context(), &pb.UploadServerDailyStatsRequest{
Stats: pbServerStats,
DomainStats: pbDomainStats,
})

View File

@@ -0,0 +1,58 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package clock
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"os/exec"
"runtime"
"time"
)
// Start TODO 需要可以在集群中配置
func Start() {
// sync once
err := Sync()
if err != nil {
remotelogs.Warn("CLOCK", "sync time clock failed: "+err.Error())
}
var ticker = time.NewTicker(1 * time.Hour)
for range ticker.C {
err := Sync()
if err != nil {
// ignore error
}
}
}
// Sync 自动校对时间
func Sync() error {
if runtime.GOOS != "linux" {
return nil
}
ntpdate, err := exec.LookPath("ntpdate")
if err != nil {
return nil
}
if len(ntpdate) > 0 {
return syncNtpdate(ntpdate)
}
return nil
}
func syncNtpdate(ntpdate string) error {
var cmd = exec.Command(ntpdate, "pool.ntp.org")
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New(err.Error() + ": " + stderr.String())
}
return nil
}

155
internal/utils/dbs/batch.go Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package dbs
import (
"database/sql"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"time"
)
type batchItem struct {
query string
args []any
}
type Batch struct {
db *sql.DB
n int
enableStat bool
onFail func(err error)
queue chan *batchItem
close chan bool
isClosed bool
}
func NewBatch(db *sql.DB, n int) *Batch {
return &Batch{
db: db,
n: n,
queue: make(chan *batchItem),
close: make(chan bool, 1),
}
}
func (this *Batch) EnableStat(b bool) {
this.enableStat = b
}
func (this *Batch) OnFail(callback func(err error)) {
this.onFail = callback
}
func (this *Batch) Add(query string, args ...any) {
this.queue <- &batchItem{
query: query,
args: args,
}
}
func (this *Batch) Exec() {
var n = this.n
if n <= 0 {
n = 4
}
var ticker = time.NewTicker(100 * time.Millisecond)
var count = 0
var lastTx *sql.Tx
For:
for {
// closed
if this.isClosed {
return
}
select {
case item := <-this.queue:
if lastTx == nil {
lastTx = this.beginTx()
if lastTx == nil {
continue For
}
}
err := this.execItem(lastTx, item)
if err != nil {
this.processErr(item.query, err)
}
count++
if count == n {
count = 0
err = lastTx.Commit()
lastTx = nil
if err != nil {
this.processErr("commit", err)
}
}
case <-ticker.C:
if lastTx == nil || count == 0 {
continue For
}
count = 0
err := lastTx.Commit()
lastTx = nil
if err != nil {
this.processErr("commit", err)
}
case <-this.close:
// closed
if lastTx != nil {
_ = lastTx.Commit()
lastTx = nil
}
return
}
}
}
func (this *Batch) Close() {
this.isClosed = true
select {
case this.close <- true:
default:
}
}
func (this *Batch) beginTx() *sql.Tx {
tx, err := this.db.Begin()
if err != nil {
this.processErr("begin transaction", err)
return nil
}
return tx
}
func (this *Batch) execItem(tx *sql.Tx, item *batchItem) error {
if this.enableStat {
defer SharedQueryStatManager.AddQuery(item.query).End()
}
_, err := tx.Exec(item.query, item.args...)
return err
}
func (this *Batch) processErr(prefix string, err error) {
if err == nil {
return
}
if this.onFail != nil {
this.onFail(err)
} else {
remotelogs.Error("SQLITE_BATCH", prefix+": "+err.Error())
}
}

View File

@@ -80,3 +80,7 @@ func (this *DB) Close() error {
events.Remove(fmt.Sprintf("db_%p", this))
return this.rawDB.Close()
}
func (this *DB) RawDB() *sql.DB {
return this.rawDB
}

View File

@@ -77,6 +77,12 @@ func (this *List) Remove(itemId uint64) {
this.removeItem(itemId)
}
func (this *List) ExpiresAt(itemId uint64) int64 {
this.locker.Lock()
defer this.locker.Unlock()
return this.itemsMap[itemId]
}
func (this *List) GC(timestamp int64) ItemMap {
if this.lastTimestamp > timestamp+1 {
return nil

View File

@@ -25,7 +25,7 @@ func (this *AllowAction) WillChange() bool {
return true
}
func (this *AllowAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *AllowAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
// do nothing
return true
return true, false
}

View File

@@ -58,11 +58,11 @@ func (this *BlockAction) WillChange() bool {
return true
}
func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
// 加入到黑名单
var timeout = this.Timeout
if timeout <= 0 {
timeout = 60 // 默认封锁60秒
timeout = 300 // 默认封锁300秒
}
SharedIPBlackList.RecordIP(IPTypeAll, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(timeout), waf.Id, waf.UseLocalFirewall, group.Id, set.Id, "")
@@ -82,14 +82,14 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
req, err := http.NewRequest(http.MethodGet, this.URL, nil)
if err != nil {
logs.Error(err)
return false
return false, false
}
req.Header.Set("User-Agent", teaconst.GlobalProductName+"/"+teaconst.Version)
resp, err := httpClient.Do(req)
if err != nil {
logs.Error(err)
return false
return false, false
}
defer func() {
_ = resp.Body.Close()
@@ -113,11 +113,11 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
data, err := os.ReadFile(path)
if err != nil {
logs.Error(err)
return false
return false, false
}
_, _ = writer.Write(data)
}
return false
return false, false
}
if len(this.Body) > 0 {
_, _ = writer.Write([]byte(this.Body))
@@ -126,5 +126,5 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
}
}
return false
return false, false
}

View File

@@ -98,10 +98,10 @@ func (this *CaptchaAction) WillChange() bool {
return true
}
func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
// 是否在白名单中
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, req.WAFServerId(), req.WAFRemoteIP()) {
return true
return true, false
}
var refURL = req.WAFRaw().URL.String()
@@ -128,7 +128,7 @@ func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
info, err := utils.SimpleEncryptMap(captchaConfig)
if err != nil {
remotelogs.Error("WAF_CAPTCHA_ACTION", "encode captcha config failed: "+err.Error())
return true
return true, false
}
// 占用一次失败次数
@@ -136,5 +136,5 @@ func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
http.Redirect(writer, req.WAFRaw(), CaptchaPath+"?info="+url.QueryEscape(info), http.StatusTemporaryRedirect)
return false
return false, false
}

View File

@@ -41,15 +41,15 @@ func (this *Get302Action) WillChange() bool {
return true
}
func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
// 仅限于Get
if request.WAFRaw().Method != http.MethodGet {
return true
return true, false
}
// 是否已经在白名单中
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
return true
return true, false
}
var m = maps.Map{
@@ -64,7 +64,7 @@ func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
info, err := utils.SimpleEncryptMap(m)
if err != nil {
remotelogs.Error("WAF_GET_302_ACTION", "encode info failed: "+err.Error())
return true
return true, false
}
http.Redirect(writer, request.WAFRaw(), Get302Path+"?info="+url.QueryEscape(info), http.StatusFound)
@@ -73,5 +73,5 @@ func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
_ = this.CloseConn(writer)
}
return false
return false, false
}

View File

@@ -1,8 +1,8 @@
package waf
import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"net/http"
)
@@ -29,20 +29,20 @@ func (this *GoGroupAction) WillChange() bool {
return true
}
func (this *GoGroupAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *GoGroupAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
nextGroup := waf.FindRuleGroup(types.Int64(this.GroupId))
if nextGroup == nil || !nextGroup.IsOn {
return true
return true, true
}
b, _, nextSet, err := nextGroup.MatchRequest(request)
if err != nil {
logs.Error(err)
return true
remotelogs.Error("WAF", "GO_GROUP_ACTION: "+err.Error())
return true, false
}
if !b {
return true
return true, false
}
return nextSet.PerformActions(waf, nextGroup, request, writer)

View File

@@ -1,8 +1,8 @@
package waf
import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"net/http"
)
@@ -30,23 +30,23 @@ func (this *GoSetAction) WillChange() bool {
return true
}
func (this *GoSetAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *GoSetAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
nextGroup := waf.FindRuleGroup(types.Int64(this.GroupId))
if nextGroup == nil || !nextGroup.IsOn {
return true
return true, true
}
nextSet := nextGroup.FindRuleSet(types.Int64(this.SetId))
if nextSet == nil || !nextSet.IsOn {
return true
return true, true
}
b, _, err := nextSet.MatchRequest(request)
if err != nil {
logs.Error(err)
return true
remotelogs.Error("WAF", "GO_GROUP_ACTION: "+err.Error())
return true, false
}
if !b {
return true
return true, false
}
return nextSet.PerformActions(waf, nextGroup, request, writer)
}

View File

@@ -27,5 +27,5 @@ type ActionInterface interface {
WillChange() bool
// Perform the action
Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool)
Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool)
}

View File

@@ -0,0 +1,133 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package waf
import (
"crypto/md5"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
"github.com/iwind/TeaGo/types"
"net/http"
"strings"
"time"
)
type JSCookieAction struct {
BaseAction
Life int32 `yaml:"life" json:"life"`
MaxFails int `yaml:"maxFails" json:"maxFails"` // 最大失败次数
FailBlockTimeout int `yaml:"failBlockTimeout" json:"failBlockTimeout"` // 失败拦截时间
Scope string `yaml:"scope" json:"scope"`
}
func (this *JSCookieAction) Init(waf *WAF) error {
this.Scope = firewallconfigs.FirewallScopeGlobal
return nil
}
func (this *JSCookieAction) Code() string {
return ActionJavascriptCookie
}
func (this *JSCookieAction) IsAttack() bool {
return false
}
func (this *JSCookieAction) WillChange() bool {
return true
}
func (this *JSCookieAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
// 是否在白名单中
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, req.WAFServerId(), req.WAFRemoteIP()) {
return true, false
}
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return true, false
}
var life = this.Life
if life <= 0 {
life = 3600
}
// 检查Cookie
var cookieName = "ge_js_validator_" + types.String(set.Id)
cookie, err := req.WAFRaw().Cookie(cookieName)
if err == nil && cookie != nil {
var cookieValue = cookie.Value
if len(cookieValue) > 10 {
var pieces = strings.Split(cookieValue, "@")
if len(pieces) == 3 {
var timestamp = pieces[0]
var sum = pieces[2]
if types.Int64(timestamp) >= time.Now().Unix()-int64(life) && fmt.Sprintf("%x", md5.Sum([]byte(timestamp+"@"+types.String(set.Id)+"@"+nodeConfig.NodeId))) == sum {
return true, false
}
}
}
}
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
writer.Header().Set("Cache-Control", "no-cache")
var timestamp = types.String(time.Now().Unix())
var cookieValue = timestamp + "@" + types.String(set.Id) + "@" + fmt.Sprintf("%x", md5.Sum([]byte(timestamp+"@"+types.String(set.Id)+"@"+nodeConfig.NodeId)))
_, _ = writer.Write([]byte(`<!DOCTYPE html>
<html>
<head>
<title></title>
<meta charset="UTF-8"/>
<script type="text/javascript">
document.cookie = "` + cookieName + `=` + cookieValue + `; path=/; max-age=` + types.String(life) + `;";
window.location.reload();
</script>
</head>
<body>
</body>
</html>`))
// 记录失败次数
this.increaseFails(req, waf.Id, group.Id, set.Id)
return false, false
}
func (this *JSCookieAction) increaseFails(req requests.Request, policyId int64, groupId int64, setId int64) (goNext bool) {
var maxFails = this.MaxFails
var failBlockTimeout = this.FailBlockTimeout
if maxFails <= 0 {
maxFails = 10 // 默认10次
} else if maxFails <= 3 {
maxFails = 3 // 不能小于3防止意外刷新出现
}
if failBlockTimeout <= 0 {
failBlockTimeout = 1800 // 默认1800s
}
var key = "JS_COOKIE:FAILS:" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + req.WAFRaw().URL.String()
var countFails = ttlcache.SharedCache.IncreaseInt64(key, 1, time.Now().Unix()+300, true)
if int(countFails) >= maxFails {
var useLocalFirewall = false
if this.Scope == firewallconfigs.FirewallScopeGlobal {
useLocalFirewall = true
}
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, useLocalFirewall, groupId, setId, "JS_COOKIE验证连续失败超过"+types.String(maxFails)+"次")
return false
}
return true
}

View File

@@ -25,6 +25,6 @@ func (this *LogAction) WillChange() bool {
return false
}
func (this *LogAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
return true
func (this *LogAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
return true, false
}

View File

@@ -34,7 +34,7 @@ func init() {
}
for task := range notifyChan {
_, err = rpcClient.FirewallRPC().NotifyHTTPFirewallEvent(rpcClient.Context(), &pb.NotifyHTTPFirewallEventRequest{
_, err = rpcClient.FirewallRPC.NotifyHTTPFirewallEvent(rpcClient.Context(), &pb.NotifyHTTPFirewallEventRequest{
ServerId: task.ServerId,
HttpFirewallPolicyId: task.HttpFirewallPolicyId,
HttpFirewallRuleGroupId: task.HttpFirewallRuleGroupId,
@@ -71,7 +71,7 @@ func (this *NotifyAction) WillChange() bool {
}
// Perform the action
func (this *NotifyAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *NotifyAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
select {
case notifyChan <- &notifyTask{
ServerId: request.WAFServerId(),
@@ -84,5 +84,5 @@ func (this *NotifyAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
}
return true
return true, false
}

View File

@@ -32,10 +32,10 @@ func (this *PageAction) WillChange() bool {
}
// Perform the action
func (this *PageAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *PageAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
writer.WriteHeader(this.Status)
_, _ = writer.Write([]byte(request.Format(this.Body)))
return false
return false, false
}

View File

@@ -33,17 +33,17 @@ func (this *Post307Action) WillChange() bool {
return true
}
func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
var cookieName = "WAF_VALIDATOR_ID"
// 仅限于POST
if request.WAFRaw().Method != http.MethodPost {
return true
return true, false
}
// 是否已经在白名单中
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
return true
return true, false
}
// 判断是否有Cookie
@@ -57,7 +57,7 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
}
var setId = m.GetString("setId")
SharedIPWhiteList.RecordIP("set:"+setId, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+life, m.GetInt64("policyId"), false, m.GetInt64("groupId"), m.GetInt64("setId"), "")
return true
return true, false
}
}
@@ -73,7 +73,7 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
info, err := utils.SimpleEncryptMap(m)
if err != nil {
remotelogs.Error("WAF_POST_302_ACTION", "encode info failed: "+err.Error())
return true
return true, false
}
// 设置Cookie
@@ -90,5 +90,5 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
_ = this.CloseConn(writer)
}
return false
return false, false
}

View File

@@ -50,7 +50,7 @@ func init() {
if len(reason) == 0 {
reason = "触发WAF规则自动加入"
}
_, err = rpcClient.IPItemRPC().CreateIPItem(rpcClient.Context(), &pb.CreateIPItemRequest{
_, err = rpcClient.IPItemRPC.CreateIPItem(rpcClient.Context(), &pb.CreateIPItemRequest{
IpListId: task.listId,
IpFrom: task.ip,
IpTo: "",
@@ -99,10 +99,10 @@ func (this *RecordIPAction) WillChange() bool {
return this.Type == "black"
}
func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
// 是否在本地白名单中
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
return true
return true, false
}
timeout := this.Timeout
@@ -147,5 +147,5 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
}
}
return this.Type != "black"
return this.Type != "black", false
}

View File

@@ -27,6 +27,6 @@ func (this *TagAction) WillChange() bool {
return false
}
func (this *TagAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
return true
func (this *TagAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
return true, true
}

View File

@@ -5,18 +5,19 @@ import "reflect"
type ActionString = string
const (
ActionLog ActionString = "log" // allow and log
ActionBlock ActionString = "block" // block
ActionCaptcha ActionString = "captcha" // block and show captcha
ActionNotify ActionString = "notify" // 告警
ActionGet302 ActionString = "get_302" // 针对GET的302重定向认证
ActionPost307 ActionString = "post_307" // 针对POST的307重定向认证
ActionRecordIP ActionString = "record_ip" // 记录IP
ActionTag ActionString = "tag" // 标签
ActionPage ActionString = "page" // 显示网页
ActionAllow ActionString = "allow" // allow
ActionGoGroup ActionString = "go_group" // go to next rule group
ActionGoSet ActionString = "go_set" // go to next rule set
ActionLog ActionString = "log" // allow and log
ActionBlock ActionString = "block" // block
ActionCaptcha ActionString = "captcha" // block and show captcha
ActionJavascriptCookie ActionString = "js_cookie" // js cookie
ActionNotify ActionString = "notify" // 告警
ActionGet302 ActionString = "get_302" // 针对GET的302重定向认证
ActionPost307 ActionString = "post_307" // 针对POST的307重定向认证
ActionRecordIP ActionString = "record_ip" // 记录IP
ActionTag ActionString = "tag" // 标签
ActionPage ActionString = "page" // 显示网页
ActionAllow ActionString = "allow" // allow
ActionGoGroup ActionString = "go_group" // go to next rule group
ActionGoSet ActionString = "go_set" // go to next rule set
)
var AllActions = []*ActionDefinition{
@@ -44,6 +45,12 @@ var AllActions = []*ActionDefinition{
Instance: new(CaptchaAction),
Type: reflect.TypeOf(new(CaptchaAction)).Elem(),
},
{
Name: "JS Cookie验证",
Code: ActionJavascriptCookie,
Instance: new(JSCookieAction),
Type: reflect.TypeOf(new(JSCookieAction)).Elem(),
},
{
Name: "告警",
Code: ActionNotify,

View File

@@ -34,7 +34,7 @@ func CaptchaIncreaseFails(req requests.Request, actionConfig *CaptchaAction, pol
useLocalFirewall = true
}
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, useLocalFirewall, groupId, setId, "CAPTCHA验证连续失败")
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, useLocalFirewall, groupId, setId, "CAPTCHA验证连续失败超过"+types.String(maxFails)+"次")
return false
}
}
@@ -50,5 +50,5 @@ func CaptchaDeleteCacheKey(req requests.Request) {
// CaptchaCacheKey 获取Captcha缓存Key
func CaptchaCacheKey(req requests.Request, pageCode CaptchaPageCode) string {
return "CAPTCHA:FAILS:" + pageCode + ":" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId())
return "CAPTCHA:FAILS:" + pageCode + ":" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + req.WAFRaw().URL.String()
}

View File

@@ -4,6 +4,7 @@ package waf
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
"github.com/TeaOSLab/EdgeNode/internal/utils/expires"
"github.com/iwind/TeaGo/types"
@@ -47,7 +48,7 @@ func NewIPList(listType IPListType) *IPList {
list.expireList = e
e.OnGC(func(itemId uint64) {
list.remove(itemId)
list.remove(itemId) // TODO 使用异步防止阻塞GC
})
return list
@@ -67,6 +68,14 @@ func (this *IPList) Add(ipType string, scope firewallconfigs.FirewallScope, serv
var id = this.nextId()
this.expireList.Add(id, expiresAt)
this.locker.Lock()
// 删除以前
oldId, ok := this.ipMap[ip]
if ok {
delete(this.idMap, oldId)
this.expireList.Remove(oldId)
}
this.ipMap[ip] = id
this.idMap[id] = ip
this.locker.Unlock()
@@ -115,6 +124,9 @@ func (this *IPList) RecordIP(ipType string,
_ = firewalls.Firewall().DropSourceIP(ip, int(seconds), true)
}
}
// 关闭此IP相关连接
conns.SharedMap.CloseIPConns(ip)
}
}
@@ -135,13 +147,52 @@ func (this *IPList) Contains(ipType string, scope firewallconfigs.FirewallScope,
return ok
}
// ContainsExpires 判断是否有某个IP并返回过期时间
func (this *IPList) ContainsExpires(ipType string, scope firewallconfigs.FirewallScope, serverId int64, ip string) (expiresAt int64, ok bool) {
switch scope {
case firewallconfigs.FirewallScopeGlobal:
ip = "*@" + ip + "@" + ipType
case firewallconfigs.FirewallScopeService:
ip = types.String(serverId) + "@" + ip + "@" + ipType
default:
ip = "*@" + ip + "@" + ipType
}
this.locker.RLock()
id, ok := this.ipMap[ip]
if ok {
expiresAt = this.expireList.ExpiresAt(id)
}
this.locker.RUnlock()
return expiresAt, ok
}
// RemoveIP 删除IP
func (this *IPList) RemoveIP(ip string, serverId int64, shouldExecute bool) {
this.locker.Lock()
delete(this.ipMap, "*@"+ip+"@"+IPTypeAll)
if serverId > 0 {
delete(this.ipMap, types.String(serverId)+"@"+ip+"@"+IPTypeAll)
{
var key = "*@" + ip + "@" + IPTypeAll
id, ok := this.ipMap[key]
if ok {
delete(this.ipMap, key)
delete(this.idMap, id)
this.expireList.Remove(id)
}
}
if serverId > 0 {
var key = types.String(serverId) + "@" + ip + "@" + IPTypeAll
id, ok := this.ipMap[key]
if ok {
delete(this.ipMap, key)
delete(this.idMap, id)
this.expireList.Remove(id)
}
}
this.locker.Unlock()
// 从本地防火墙中删除

View File

@@ -6,6 +6,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/iwind/TeaGo/assert"
"github.com/iwind/TeaGo/logs"
timeutil "github.com/iwind/TeaGo/utils/time"
"runtime"
"strconv"
"testing"
@@ -13,18 +14,34 @@ import (
)
func TestNewIPList(t *testing.T) {
list := NewIPList(IPListTypeDeny)
var list = NewIPList(IPListTypeDeny)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix())
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.2", time.Now().Unix()+1)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix()+2)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeService, 1, "127.0.0.3", time.Now().Unix()+3)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.10", time.Now().Unix()+10)
list.RemoveIP("127.0.0.1", 1, false)
logs.PrintAsJSON(list.ipMap, t)
logs.PrintAsJSON(list.idMap, t)
}
func TestIPList_Expire(t *testing.T) {
var list = NewIPList(IPListTypeDeny)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix())
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.2", time.Now().Unix()+1)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix()+2)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.3", time.Now().Unix()+3)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.10", time.Now().Unix()+10)
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.10", time.Now().Unix()+6)
var ticker = time.NewTicker(1 * time.Second)
for range ticker.C {
t.Log("====")
list.locker.Lock()
logs.PrintAsJSON(list.ipMap, t)
logs.PrintAsJSON(list.idMap, t)
list.locker.Unlock()
if len(list.idMap) == 0 {
break
}
@@ -32,22 +49,39 @@ func TestNewIPList(t *testing.T) {
}
func TestIPList_Contains(t *testing.T) {
a := assert.NewAssertion(t)
var a = assert.NewAssertion(t)
list := NewIPList(IPListTypeDeny)
var list = NewIPList(IPListTypeDeny)
for i := 0; i < 1_0000; i++ {
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
}
//list.RemoveIP("192.168.1.100")
a.IsTrue(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1.100"))
a.IsFalse(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.2.100"))
{
a.IsTrue(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1.100"))
}
{
a.IsFalse(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.2.100"))
}
}
func TestIPList_ContainsExpires(t *testing.T) {
var list = NewIPList(IPListTypeDeny)
for i := 0; i < 1_0000; i++ {
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
}
// list.RemoveIP("192.168.1.100", 1, false)
for _, ip := range []string{"192.168.1.100", "192.168.2.100"} {
expiresAt, ok := list.ContainsExpires(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, ip)
t.Log(ok, expiresAt, timeutil.FormatTime("Y-m-d H:i:s", expiresAt))
}
}
func BenchmarkIPList_Add(b *testing.B) {
runtime.GOMAXPROCS(1)
list := NewIPList(IPListTypeDeny)
var list = NewIPList(IPListTypeDeny)
for i := 0; i < b.N; i++ {
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
}
@@ -57,7 +91,8 @@ func BenchmarkIPList_Add(b *testing.B) {
func BenchmarkIPList_Has(b *testing.B) {
runtime.GOMAXPROCS(1)
list := NewIPList(IPListTypeDeny)
var list = NewIPList(IPListTypeDeny)
b.ResetTimer()
for i := 0; i < 1_0000; i++ {
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)

View File

@@ -667,7 +667,7 @@ func (this *Rule) execFilter(value interface{}) interface{} {
}
value, goNext, err = filterInstance.Do(value, filter.Options)
if err != nil {
remotelogs.Println("WAF", "filter error: "+err.Error())
remotelogs.Error("WAF", "filter error: "+err.Error())
break
}
if !goNext {

View File

@@ -136,19 +136,19 @@ func (this *RuleSet) ActionCodes() []string {
return this.actionCodes
}
func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Request, writer http.ResponseWriter) bool {
func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
if len(waf.Mode) != 0 && waf.Mode != firewallconfigs.FirewallModeDefend {
return true
return true, false
}
// 先执行allow
for _, instance := range this.actionInstances {
if !instance.WillChange() {
goNext := req.WAFOnAction(instance)
if !goNext {
return false
continueRequest = req.WAFOnAction(instance)
if !continueRequest {
return false, false
}
instance.Perform(waf, group, this, req, writer)
_, goNextSet = instance.Perform(waf, group, this, req, writer)
}
}
@@ -156,15 +156,15 @@ func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Req
for _, instance := range this.actionInstances {
// 只执行第一个可能改变请求的动作,其余的都会被忽略
if instance.WillChange() {
goNext := req.WAFOnAction(instance)
if !goNext {
return false
continueRequest = req.WAFOnAction(instance)
if !continueRequest {
return false, false
}
return instance.Perform(waf, group, this, req, writer)
}
}
return true
return true, goNextSet
}
func (this *RuleSet) MatchRequest(req requests.Request) (b bool, hasRequestBody bool, err error) {

View File

@@ -241,7 +241,7 @@ func (this *WAF) MoveOutboundRuleGroup(fromIndex int, toIndex int) {
this.Outbound = result
}
func (this *WAF) MatchRequest(req requests.Request, writer http.ResponseWriter) (goNext bool, hasRequestBody bool, group *RuleGroup, set *RuleSet, err error) {
func (this *WAF) MatchRequest(req requests.Request, writer http.ResponseWriter) (goNext bool, hasRequestBody bool, group *RuleGroup, sets *RuleSet, err error) {
if !this.hasInboundRules {
return true, hasRequestBody, nil, nil, nil
}
@@ -272,8 +272,10 @@ func (this *WAF) MatchRequest(req requests.Request, writer http.ResponseWriter)
return true, hasRequestBody, nil, nil, err
}
if b {
goNext = set.PerformActions(this, group, req, writer)
return goNext, hasRequestBody, group, set, nil
continueRequest, goNextSet := set.PerformActions(this, group, req, writer)
if !goNextSet {
return continueRequest, hasRequestBody, group, set, nil
}
}
}
return true, hasRequestBody, nil, nil, nil
@@ -296,8 +298,10 @@ func (this *WAF) MatchResponse(req requests.Request, rawResp *http.Response, wri
return true, hasRequestBody, nil, nil, err
}
if b {
goNext = set.PerformActions(this, group, req, writer)
return goNext, hasRequestBody, group, set, nil
continueRequest, goNextSet := set.PerformActions(this, group, req, writer)
if !goNextSet {
return continueRequest, hasRequestBody, group, set, nil
}
}
}
return true, hasRequestBody, nil, nil, nil