Compare commits

...

73 Commits

Author SHA1 Message Date
刘祥超
57cb00edf0 修复无法添加IP到本地防火墙的问题 2022-09-04 17:06:19 +08:00
刘祥超
3fb39b479a 优化IP名单锁 2022-09-03 21:10:13 +08:00
刘祥超
4a1daff143 增加简化的缓存条件设置 2022-09-03 18:16:35 +08:00
刘祥超
dd1dbd424e 修复WAF IP名单测试用例的一个可能异常 2022-09-03 09:56:36 +08:00
刘祥超
305cb4b46e 优化WAF中IP名单 2022-09-03 09:54:25 +08:00
刘祥超
ef90dce29b 优化节点进程退出逻辑 2022-09-02 16:12:58 +08:00
刘祥超
3cb69f4c71 将IP加入黑名单时,同时也会关闭此IP相关的连接 2022-09-02 15:20:58 +08:00
刘祥超
af4cd05df2 增加一处注释 2022-09-01 09:06:21 +08:00
刘祥超
64e0ae80b7 DDoS防护增加秒级连接速率限制 2022-08-31 10:00:38 +08:00
刘祥超
8bba228745 优化代码 2022-08-30 18:49:21 +08:00
刘祥超
8cc06e6707 鉴权成功时也在访问日志中记录鉴权类型 2022-08-30 17:28:58 +08:00
刘祥超
52fdee2eeb 修复访问控制后缓存Key包含认证参数的问题 2022-08-30 12:04:01 +08:00
刘祥超
b5f52dd136 优化鉴权 2022-08-30 11:24:23 +08:00
刘祥超
abda886de5 如果系统安装了ntpdate,则自动尝试利用ntpdate同步时间 2022-08-27 15:28:53 +08:00
刘祥超
18f08525b9 WAF和其他请求关闭连接时更加快速 2022-08-27 10:49:16 +08:00
刘祥超
b2a9a31fe5 增加edge-node bandwidth命令查看实时带宽 2022-08-26 16:47:46 +08:00
刘祥超
f578114aeb 修复HTTPS连接无法记录带宽的问题,优化带宽计算方法 2022-08-26 16:47:42 +08:00
刘祥超
bf4f47fc35 DDoS防护增加单IP TCP新连接速率黑名单 2022-08-26 11:31:43 +08:00
刘祥超
0be951742a WAF优化captcha和js_cookie的失败计数器/增强js_cookie的安全性 2022-08-25 17:06:52 +08:00
刘祥超
59d3d6ae4b WAF标签动作匹配之后可以继续尝试匹配别的分组中的规则集 2022-08-25 16:44:44 +08:00
刘祥超
d061876f7e 增加Javascript Cookie验证 2022-08-25 15:35:32 +08:00
刘祥超
ddaec82415 优化RPC获取服务实例方式 2022-08-24 20:04:46 +08:00
刘祥超
8afd00f00d 删除HTTP/2自动关闭Body的逻辑 2022-08-24 16:43:32 +08:00
刘祥超
0b306f0a22 忽略部分HTTP/2错误提示 2022-08-24 16:30:38 +08:00
刘祥超
b36a36172b 优化代码 2022-08-23 14:53:39 +08:00
刘祥超
770278bbbc 实现IP库检查更新 2022-08-23 14:32:39 +08:00
刘祥超
ca24818571 优化缓存索引内存使用 2022-08-22 09:44:09 +08:00
刘祥超
cd0af22655 优化代码 2022-08-22 08:31:39 +08:00
刘祥超
96cb8d8af7 IP库改为手动初始化 2022-08-21 23:09:47 +08:00
刘祥超
91face15bf 使用新版IP库 2022-08-21 20:37:49 +08:00
刘祥超
6f52df63a5 只有系统内存超过3GB的才缓存Hash 2022-08-20 12:09:15 +08:00
刘祥超
509d81dc66 修复检测是否为高速硬盘只能检查sda/vda设备的问题 2022-08-20 11:59:49 +08:00
刘祥超
817f2a6f91 大幅提升缓存索引性能 2022-08-20 11:47:57 +08:00
刘祥超
d74e10c7a8 缓存任务忽略连接API错误 2022-08-20 08:15:16 +08:00
刘祥超
1a1280b76e 优化代码 2022-08-19 14:50:26 +08:00
刘祥超
85d3b1169f 修复Gif中透明图转换WebP失败的问题 2022-08-19 13:39:46 +08:00
刘祥超
1b32292e0c WebP压缩Gif时,自动跳过转换失败的帧,并只提示最后一次错误 2022-08-19 13:27:18 +08:00
刘祥超
e6ba44fb2f [SQLITE]使用事务批量提交一些缓存相关任务 2022-08-17 21:04:00 +08:00
刘祥超
9b6e022f46 将版本修改为0.5.2 2022-08-17 18:56:48 +08:00
刘祥超
4e9ab19fc0 修复因为fsnotify包引用而无法编译的问题 2022-08-17 12:12:40 +08:00
刘祥超
4d634d8fa5 修复一处HTTPS错误提示 2022-08-15 18:26:23 +08:00
刘祥超
a538282e4f 版本修改为0.5.1 2022-08-15 18:26:10 +08:00
刘祥超
df31921954 优化代码 2022-08-14 16:28:40 +08:00
刘祥超
299abb7b04 缓存本地数据库发生错误时同时提示数据库文件名 2022-08-14 15:17:07 +08:00
刘祥超
85ce63b4d3 优化API命名/优化一处测试用例 2022-08-11 12:44:27 +08:00
刘祥超
9d9ae288bd 优化代码 2022-08-10 14:37:27 +08:00
刘祥超
c0a35eb5e7 优化代码 2022-08-09 22:59:37 +08:00
刘祥超
d1d0ff062b 增加Ln节点向源站的最大连接数 2022-08-08 08:12:49 +08:00
刘祥超
396e8a22c4 改进一处错误提示 2022-08-07 19:02:06 +08:00
刘祥超
4f040db1ef 版本号改为0.5.0 2022-08-07 19:01:45 +08:00
刘祥超
e3cf111344 缓存条件增加If-None-Match和If-Modified-Since是否回源选项;默认不回源,防止源站返回304 2022-08-07 16:37:29 +08:00
刘祥超
28cb3c383d 修复一处注释 2022-08-07 11:18:16 +08:00
刘祥超
10665c0f37 优化代码 2022-08-07 11:12:29 +08:00
刘祥超
4096f11909 修复一处测试用例 2022-08-06 20:52:52 +08:00
刘祥超
5df209b6d5 优化代码 2022-08-04 11:34:06 +08:00
刘祥超
db353fe025 nftables封禁IP使用异步操作 2022-08-04 11:01:16 +08:00
刘祥超
30c3e143b8 edge-node pprof命令增加--addr参数 2022-08-04 10:18:23 +08:00
刘祥超
15fe7b33a4 新增屏蔽错误 2022-08-04 10:04:26 +08:00
刘祥超
17e0666ba4 优化代码 2022-08-03 23:31:08 +08:00
刘祥超
3f24bfaaf5 HTTP读取L2节点时增加最大连接数、最大空闲连接数、最大空闲时间 2022-08-03 20:54:08 +08:00
刘祥超
ceadcfece9 HTTP源站有多个L2节点时发生错误时会主动尝试下一个L2节点 2022-08-03 20:30:59 +08:00
刘祥超
bc706237ef HTTP源站读取错误时自动尝试下一个源站 2022-08-03 19:33:50 +08:00
刘祥超
b8c5a78f2e TCP/TLS/UDP第一次连接源站失败后,自动尝试下一个源站 2022-08-03 16:09:47 +08:00
刘祥超
9ad1c3a3c8 TLS支持默认SNI回源 2022-08-03 11:04:58 +08:00
刘祥超
8cfde43f5d TCP连接源站失败时关闭连接/TCP连接也支持备用源站 2022-08-01 19:28:27 +08:00
刘祥超
6f843b071a 执行IP名单更新任务时防止阻塞/优化节点升级代码 2022-08-01 18:54:54 +08:00
刘祥超
a72b025900 优化忽略客户端关闭连接错误条件 2022-08-01 16:53:08 +08:00
刘祥超
dfb66775d7 40x, 50x提示默认使用HTML;50x提示增加原因信息 2022-07-30 10:48:41 +08:00
刘祥超
c4dca2df30 UDP也记录带宽 2022-07-30 09:30:05 +08:00
刘祥超
a3525bdaa4 修复节点自动升级时无法自动启动的Bug 2022-07-28 14:38:08 +08:00
刘祥超
09390bbb97 优化代码 2022-07-27 18:10:43 +08:00
刘祥超
70ae4391d7 减少Daemon使用的内存 2022-07-26 09:41:43 +08:00
刘祥超
85931f55e1 修改版本号为0.4.11 2022-07-26 09:33:38 +08:00
147 changed files with 2612 additions and 2391 deletions

View File

@@ -52,7 +52,6 @@ function build() {
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
# we support TOA on linux/amd64 only
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

View File

@@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"flag"
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/apps"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
@@ -19,7 +20,7 @@ import (
)
func main() {
app := apps.NewAppCmd().
var app = apps.NewAppCmd().
Version(teaconst.Version).
Product(teaconst.ProductName).
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog]").
@@ -67,24 +68,30 @@ func main() {
fmt.Println("done")
})
app.On("pprof", func() {
// TODO 自己指定端口
addr := "127.0.0.1:6060"
var flagSet = flag.NewFlagSet("pprof", flag.ExitOnError)
var addr string
flagSet.StringVar(&addr, "addr", "", "")
_ = flagSet.Parse(os.Args[2:])
if len(addr) == 0 {
addr = "127.0.0.1:6060"
}
logs.Println("starting with pprof '" + addr + "'...")
go func() {
err := http.ListenAndServe(addr, nil)
if err != nil {
logs.Println("[error]" + err.Error())
logs.Println("[ERROR]" + err.Error())
}
}()
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
app.On("dbstat", func() {
teaconst.EnableDBStat = true
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
app.On("trackers", func() {
@@ -154,7 +161,7 @@ func main() {
app.On("ip.drop", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS]")
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS] [--async]")
return
}
var ip = args[0]
@@ -168,6 +175,11 @@ func main() {
if ok {
timeoutSeconds = types.Int(timeout[0])
}
var async = false
_, ok = options["async"]
if ok {
async = true
}
fmt.Println("drop ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
var sock = gosock.NewTmpSock(teaconst.ProcessName)
@@ -176,6 +188,7 @@ func main() {
Params: map[string]interface{}{
"ip": ip,
"timeoutSeconds": timeoutSeconds,
"async": async,
},
})
if err != nil {
@@ -305,8 +318,23 @@ func main() {
}
}
})
app.On("bandwidth", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
var statsMap = maps.NewMap(reply.Params).Get("stats")
statsJSON, err := json.MarshalIndent(statsMap, "", " ")
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
fmt.Println(string(statsJSON))
})
app.Run(func() {
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
}

15
go.mod
View File

@@ -4,6 +4,7 @@ go 1.18
replace (
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
)
require (
@@ -16,10 +17,10 @@ require (
github.com/go-redis/redis/v8 v8.11.5
github.com/golang/protobuf v1.5.2
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5
github.com/klauspost/compress v1.15.8
github.com/mattn/go-sqlite3 v1.14.9
github.com/mdlayher/netlink v1.4.2
@@ -27,20 +28,22 @@ require (
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
github.com/shirou/gopsutil/v3 v3.22.2
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
golang.org/x/text v0.3.7
google.golang.org/grpc v1.45.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
rogchap.com/v8go v0.7.0
)
require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/webp v1.1.0 // indirect
github.com/chai2010/webp v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect

84
go.sum
View File

@@ -1,17 +1,13 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -20,8 +16,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -37,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -48,10 +43,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -83,20 +76,20 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55 h1:shQNx0flJFBwKsGE7Hs3bI2bDz+YF0zl/4qE8B2KRiY=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5 h1:SgKhrqRhWVpu0ZKxQM8MGjdhy7hlH3Xax8snwsZWSic=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -109,14 +102,10 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -150,23 +139,11 @@ github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb h1:2dC7L10LmTqlyMV
github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -180,9 +157,7 @@ github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtS
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
@@ -196,41 +171,30 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -259,21 +223,15 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -300,11 +258,10 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -316,7 +273,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
@@ -330,7 +286,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
@@ -339,7 +294,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
@@ -358,25 +312,19 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
rogchap.com/v8go v0.7.0 h1:kgjbiO4zE5itA962ze6Hqmbs4HgZbGzmueCXsZtremg=
rogchap.com/v8go v0.7.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs=

View File

@@ -89,7 +89,10 @@ func (this *LogWriter) Write(message string) {
}
}
this.c <- message
select {
case this.c <- message:
default:
}
}
func (this *LogWriter) Close() {

View File

@@ -96,13 +96,13 @@ func (this *FileList) Reset() error {
}
func (this *FileList) Add(hash string, item *Item) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
err := db.Add(hash, item)
err := db.AddAsync(hash, item)
if err != nil {
return err
}
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
}
func (this *FileList) Exist(hash string) (bool, error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// 如果Hash列表里不存在那么必然不存在
if !db.hashMap.Exist(hash) {
return false, nil
}
var item = this.memoryCache.Read(hash)
if item != nil {
return true, nil
@@ -225,9 +230,9 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
return err
}
if notFound {
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return err
return db.WrapError(err)
}
}
@@ -266,7 +271,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
_ = check
row := db.statStmt.QueryRow()
var row = db.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
@@ -291,13 +296,13 @@ func (this *FileList) Count() (int64, error) {
// IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
return db.IncreaseHit(hash)
return db.IncreaseHitAsync(hash)
}
// OnAdd 添加事件
@@ -326,17 +331,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
return fnv.HashString(hash) % CountFileDB
}
func (this *FileList) getDB(hash string) *FileListDB {
func (this *FileList) GetDB(hash string) *FileListDB {
return this.dbList[fnv.HashString(hash)%CountFileDB]
}
func (this *FileList) remove(hash string) (notFound bool, err error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// HashMap中不存在则确定不存在
if !db.hashMap.Exist(hash) {
return true, nil
}
defer db.hashMap.Delete(hash)
// 从缓存中删除
this.memoryCache.Delete(hash)
@@ -357,16 +368,16 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
return false, err
}
_, err = db.deleteByHashStmt.Exec(hash)
err = db.DeleteAsync(hash)
if err != nil {
return false, err
return false, db.WrapError(err)
}
atomic.AddInt64(&this.total, -1)
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return false, err
return false, db.WrapError(err)
}
if this.onRemove != nil {

View File

@@ -6,6 +6,8 @@ import (
"database/sql"
"errors"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/iwind/TeaGo/types"
@@ -16,9 +18,15 @@ import (
)
type FileListDB struct {
dbPath string
readDB *dbs.DB
writeDB *dbs.DB
writeBatch *dbs.Batch
hashMap *FileListHashMap
itemsTableName string
hitsTableName string
@@ -28,29 +36,48 @@ type FileListDB struct {
isReady bool
// cacheItems
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
selectByHashStmt *dbs.Stmt // 使用hash查询数据
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
insertSQL string
selectByHashStmt *dbs.Stmt // 使用hash查询数据
selectHashListStmt *dbs.Stmt
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
deleteByHashSQL string
statStmt *dbs.Stmt // 统计
purgeStmt *dbs.Stmt // 清理
deleteAllStmt *dbs.Stmt // 删除所有数据
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
// hits
insertHitStmt *dbs.Stmt // 写入数据
increaseHitStmt *dbs.Stmt // 增加点击量
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
insertHitSQL string // 写入数据
increaseHitSQL string // 增加点击量
deleteHitByHashSQL string // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
}
func NewFileListDB() *FileListDB {
return &FileListDB{}
return &FileListDB{
hashMap: NewFileListHashMap(),
}
}
func (this *FileListDB) Open(dbPath string) error {
this.dbPath = dbPath
// 动态调整Cache值
var cacheSize = 32000
var memoryGB = utils.SystemMemoryGB()
if memoryGB >= 8 {
cacheSize += 32000 * memoryGB / 8
}
// write db
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize)+"&_secure_delete=FAST")
if err != nil {
return errors.New("open write database failed: " + err.Error())
}
@@ -65,13 +92,22 @@ func (this *FileListDB) Open(dbPath string) error {
}**/
this.writeDB = dbs.NewDB(writeDB)
this.writeBatch = dbs.NewBatch(writeDB, 4)
this.writeBatch.OnFail(func(err error) {
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error())
})
goman.New(func() {
this.writeBatch.Exec()
})
if teaconst.EnableDBStat {
this.writeBatch.EnableStat(true)
this.writeDB.EnableStat(true)
}
// read db
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize))
if err != nil {
return errors.New("open read database failed: " + err.Error())
}
@@ -115,7 +151,8 @@ func (this *FileListDB) Init() error {
return err
}
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
if err != nil {
return err
}
@@ -125,7 +162,10 @@ func (this *FileListDB) Init() error {
return err
}
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
if err != nil {
return err
}
@@ -147,17 +187,11 @@ func (this *FileListDB) Init() error {
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
if err != nil {
return err
}
this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
@@ -166,6 +200,14 @@ func (this *FileListDB) Init() error {
this.isReady = true
// 加载HashMap
go func() {
err := this.hashMap.Load(this)
if err != nil {
remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
}
}()
return nil
}
@@ -177,17 +219,47 @@ func (this *FileListDB) Total() int64 {
return this.total
}
func (this *FileListDB) Add(hash string, item *Item) error {
func (this *FileListDB) AddAsync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
return nil
}
func (this *FileListDB) AddSync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
// 放入队列
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
if err != nil {
return this.WrapError(err)
}
return nil
}
func (this *FileListDB) DeleteAsync(hash string) error {
this.hashMap.Delete(hash)
this.writeBatch.Add(this.deleteByHashSQL, hash)
return nil
}
func (this *FileListDB) DeleteSync(hash string) error {
this.hashMap.Delete(hash)
_, err := this.deleteByHashStmt.Exec(hash)
if err != nil {
return err
}
return nil
}
@@ -246,10 +318,36 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
return
}
func (this *FileListDB) IncreaseHit(hash string) error {
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
rows, err := this.selectHashListStmt.Query(lastId)
if err != nil {
return nil, 0, err
}
var id int64
var hash string
for rows.Next() {
err = rows.Scan(&id, &hash)
if err != nil {
_ = rows.Close()
return
}
maxId = id
hashList = append(hashList, hash)
}
_ = rows.Close()
return
}
func (this *FileListDB) IncreaseHitAsync(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return err
this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
return nil
}
func (this *FileListDB) DeleteHitAsync(hash string) error {
this.writeBatch.Add(this.deleteHitByHashSQL, hash)
return nil
}
func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -262,7 +360,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
for {
result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
if err != nil {
return err
return this.WrapError(err)
}
affectedRows, err := result.RowsAffected()
if err != nil {
@@ -281,9 +379,11 @@ func (this *FileListDB) CleanAll() error {
_, err := this.deleteAllStmt.Exec()
if err != nil {
return err
return this.WrapError(err)
}
this.hashMap.Clean()
return nil
}
@@ -300,6 +400,9 @@ func (this *FileListDB) Close() error {
if this.selectByHashStmt != nil {
_ = this.selectByHashStmt.Close()
}
if this.selectHashListStmt != nil {
_ = this.selectHashListStmt.Close()
}
if this.deleteByHashStmt != nil {
_ = this.deleteByHashStmt.Close()
}
@@ -315,16 +418,6 @@ func (this *FileListDB) Close() error {
if this.listOlderItemsStmt != nil {
_ = this.listOlderItemsStmt.Close()
}
if this.insertHitStmt != nil {
_ = this.insertHitStmt.Close()
}
if this.increaseHitStmt != nil {
_ = this.increaseHitStmt.Close()
}
if this.deleteHitByHashStmt != nil {
_ = this.deleteHitByHashStmt.Close()
}
if this.lfuHitsStmt != nil {
_ = this.lfuHitsStmt.Close()
}
@@ -345,12 +438,23 @@ func (this *FileListDB) Close() error {
}
}
if this.writeBatch != nil {
this.writeBatch.Close()
}
if len(errStrings) == 0 {
return nil
}
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
}
func (this *FileListDB) WrapError(err error) error {
if err == nil {
return nil
}
return errors.New(err.Error() + "(file: " + this.dbPath + ")")
}
// 初始化
func (this *FileListDB) initTables(times int) error {
{
@@ -393,10 +497,10 @@ ON "` + this.itemsTableName + `" (
if dropErr == nil {
return this.initTables(times + 1)
}
return err
return this.WrapError(err)
}
return err
return this.WrapError(err)
}
}
@@ -421,10 +525,10 @@ ON "` + this.hitsTableName + `" (
if dropErr == nil {
return this.initTables(times + 1)
}
return err
return this.WrapError(err)
}
return err
return this.WrapError(err)
}
}

View File

@@ -0,0 +1,114 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"math/big"
"sync"
)
// FileListHashMap 文件Hash列表
type FileListHashMap struct {
m map[uint64]zero.Zero
locker sync.RWMutex
isAvailable bool
isReady bool
}
func NewFileListHashMap() *FileListHashMap {
return &FileListHashMap{
m: map[uint64]zero.Zero{},
isAvailable: false,
isReady: false,
}
}
func (this *FileListHashMap) Load(db *FileListDB) error {
// 如果系统内存过小,我们不缓存
if utils.SystemMemoryGB() < 3 {
return nil
}
this.isAvailable = true
var lastId int64
for {
hashList, maxId, err := db.ListHashes(lastId)
if err != nil {
return err
}
if len(hashList) == 0 {
break
}
this.AddHashes(hashList)
lastId = maxId
}
this.isReady = true
return nil
}
func (this *FileListHashMap) Add(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
this.m[this.bigInt(hash)] = zero.New()
this.locker.Unlock()
}
func (this *FileListHashMap) AddHashes(hashes []string) {
if !this.isAvailable {
return
}
this.locker.Lock()
for _, hash := range hashes {
this.m[this.bigInt(hash)] = zero.New()
}
this.locker.Unlock()
}
func (this *FileListHashMap) Delete(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
delete(this.m, this.bigInt(hash))
this.locker.Unlock()
}
func (this *FileListHashMap) Exist(hash string) bool {
if !this.isAvailable {
return true
}
if !this.isReady {
// 只有完全Ready时才能判断是否为false
return true
}
this.locker.RLock()
_, ok := this.m[this.bigInt(hash)]
this.locker.RUnlock()
return ok
}
func (this *FileListHashMap) Clean() {
this.locker.Lock()
this.m = map[uint64]zero.Zero{}
this.locker.Unlock()
}
func (this *FileListHashMap) IsReady() bool {
return this.isReady
}
func (this *FileListHashMap) bigInt(hash string) uint64 {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
return bigInt.Uint64()
}

View File

@@ -0,0 +1,91 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"math/big"
"runtime"
"strconv"
"testing"
)
func TestFileListHashMap_Memory(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = caches.NewFileListHashMap()
for i := 0; i < 1_000_000; i++ {
m.Add(stringutil.Md5(types.String(i)))
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_Memory2(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = map[uint64]zero.Zero{}
for i := 0; i < 1_000_000; i++ {
m[uint64(i)] = zero.New()
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_BigInt(t *testing.T) {
for _, s := range []string{"1", "2", "3", "123", "123456"} {
var hash = stringutil.Md5(s)
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
}
}
func TestFileListHashMap_Load(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
var m = caches.NewFileListHashMap()
err = m.Load(list.GetDB("abc"))
if err != nil {
t.Fatal(err)
}
m.Add("abc")
for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
t.Log(hash, "=>", m.Exist(hash))
}
}
func Benchmark_BigInt(b *testing.B) {
var hash = stringutil.Md5("123456")
b.ResetTimer()
for i := 0; i < b.N; i++ {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
_ = bigInt.Uint64()
}
}

View File

@@ -69,8 +69,8 @@ func TestFileList_Add_Many(t *testing.T) {
_ = list.Close()
}()
before := time.Now()
for i := 0; i < 100_000; i++ {
var before = time.Now()
for i := 0; i < 10_000_000; i++ {
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &caches.Item{
Key: u,
@@ -290,12 +290,12 @@ func TestFileList_Stat(t *testing.T) {
}
func TestFileList_Count(t *testing.T) {
list := caches.NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
var before = time.Now()
count, err := list.Count()
if err != nil {
t.Fatal(err)
@@ -367,6 +367,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
}

View File

@@ -6,7 +6,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"strconv"
"sync"
@@ -16,7 +15,7 @@ var SharedManager = NewManager()
func init() {
events.On(events.EventQuit, func() {
logs.Println("CACHE", "quiting cache manager")
remotelogs.Println("CACHE", "quiting cache manager")
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
})
}

View File

@@ -5,7 +5,7 @@ package caches
import (
"encoding/json"
"errors"
"io/ioutil"
"os"
)
// PartialRanges 内容分区范围定义
@@ -30,7 +30,7 @@ func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
}
func NewPartialRangesFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
@@ -116,12 +116,12 @@ func (this *PartialRanges) WriteToFile(path string) error {
if err != nil {
return errors.New("convert to json failed: " + err.Error())
}
return ioutil.WriteFile(path, data, 0666)
return os.WriteFile(path, data, 0666)
}
// ReadFromFile 从文件中读取
func (this *PartialRanges) ReadFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}

View File

@@ -65,12 +65,13 @@ var sharedWritingFileKeyLocker = sync.Mutex{}
var maxOpenFiles = NewMaxOpenFiles()
const maxOpenFilesSlowCost = 5000 * time.Microsecond // 0.5ms
const maxOpenFilesSlowCost = 1000 * time.Microsecond // us
const protectingLoadWhenDump = false
// FileStorage 文件缓存
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
//
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
type FileStorage struct {
policy *serverconfigs.HTTPCachePolicy
options *serverconfigs.HTTPFileCacheStorage // 二级缓存

View File

@@ -8,7 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"io"
"net/http"
"runtime"
"strconv"
@@ -152,7 +152,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
"Last-Modified": []string{"Wed, 06 Jan 2021 10:03:29 GMT"},
"Server": []string{"CDN-Server"},
},
Body: ioutil.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
Body: io.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
}
for k, v := range resp.Header {

View File

@@ -95,7 +95,8 @@ func (this *MemoryStorage) Init() error {
// 启动定时Flush memory to disk任务
if this.parentStorage != nil {
// TODO 应该根据磁盘性能决定线程数
var threads = 1
// TODO 线程数应该可以在缓存策略和节点中设定
var threads = runtime.NumCPU()
for i := 0; i < threads; i++ {
goman.New(func() {

View File

@@ -8,6 +8,7 @@ import (
"runtime"
"runtime/debug"
"strconv"
"sync"
"testing"
"time"
)
@@ -304,3 +305,30 @@ func TestMemoryStorage_Stop(t *testing.T) {
t.Log(len(m))
}
func BenchmarkValuesMap(b *testing.B) {
var m = map[uint64]*MemoryItem{}
var count = 1_000_000
for i := 0; i < count; i++ {
m[uint64(i)] = &MemoryItem{
ExpiresAt: time.Now().Unix(),
}
}
b.Log(len(m))
var locker = sync.Mutex{}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
locker.Lock()
_, ok := m[uint64(rands.Int(0, 1_000_000))]
_ = ok
locker.Unlock()
locker.Lock()
delete(m, uint64(rands.Int(2, 1000000)))
locker.Unlock()
}
})
}

View File

@@ -5,7 +5,6 @@ package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"testing"
"time"
@@ -16,7 +15,7 @@ func TestPartialFileWriter_Write(t *testing.T) {
_ = os.Remove(path)
var reader = func() {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedBrotliReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedDeflateReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedGzipReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedZSTDReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -5,10 +5,46 @@ package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
stringutil "github.com/iwind/TeaGo/utils/string"
"strings"
"testing"
"time"
)
func TestBrotliWriter_LargeFile(t *testing.T) {
var data = []byte{}
for i := 0; i < 1024*1024; i++ {
data = append(data, stringutil.Rand(32)...)
}
t.Log(len(data)/1024/1024, "M")
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
t.Fatal(err)
}
var offset = 0
var size = 4096
for offset < len(data) {
_, err = writer.Write(data[offset : offset+size])
if err != nil {
t.Fatal(err)
}
offset += size
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
}
func BenchmarkBrotliWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/andybalholm/brotli"
"io"
@@ -11,6 +12,10 @@ import (
var sharedBrotliWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -4,6 +4,7 @@ package compressions
import (
"compress/flate"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -11,6 +12,10 @@ import (
var sharedDeflateWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -4,6 +4,7 @@ package compressions
import (
"compress/gzip"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -11,6 +12,10 @@ import (
var sharedGzipWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/klauspost/compress/zstd"
"io"
@@ -11,6 +12,10 @@ import (
var sharedZSTDWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,7 +3,7 @@ package configs
import (
"github.com/iwind/TeaGo/Tea"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
)
// APIConfig 节点API配置
@@ -17,7 +17,7 @@ type APIConfig struct {
}
func LoadAPIConfig() (*APIConfig, error) {
data, err := ioutil.ReadFile(Tea.ConfigFile("api.yaml"))
data, err := os.ReadFile(Tea.ConfigFile("api.yaml"))
if err != nil {
return nil, err
}
@@ -37,6 +37,6 @@ func (this *APIConfig) WriteFile(path string) error {
if err != nil {
return err
}
err = ioutil.WriteFile(path, data, 0666)
err = os.WriteFile(path, data, 0666)
return err
}

117
internal/conns/map.go Normal file
View File

@@ -0,0 +1,117 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package conns
import (
"net"
"sync"
)
var SharedMap = NewMap()
type Map struct {
m map[string]map[int]net.Conn // ip => { port => Conn }
locker sync.RWMutex
}
func NewMap() *Map {
return &Map{
m: map[string]map[int]net.Conn{},
}
}
func (this *Map) Add(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
this.m[ip] = map[int]net.Conn{
port: conn,
}
} else {
connMap[port] = conn
}
}
func (this *Map) Remove(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
return
}
delete(connMap, port)
if len(connMap) == 0 {
delete(this.m, ip)
}
}
func (this *Map) CountIPConns(ip string) int {
this.locker.RLock()
var l = len(this.m[ip])
this.locker.RUnlock()
return l
}
func (this *Map) CloseIPConns(ip string) {
var conns = []net.Conn{}
this.locker.RLock()
connMap, ok := this.m[ip]
// 复制防止在Close时产生并发冲突
if ok {
for _, conn := range connMap {
conns = append(conns, conn)
}
}
// 需要在Close之前结束防止死循环
this.locker.RUnlock()
if ok {
for _, conn := range conns {
_ = conn.Close()
}
// 这里不需要从 m 中删除,因为关闭时会自然触发回调
}
}
func (this *Map) AllConns() []net.Conn {
this.locker.RLock()
defer this.locker.RUnlock()
var result = []net.Conn{}
for _, m := range this.m {
for _, conn := range m {
result = append(result, conn)
}
}
return result
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.4.10"
Version = "0.5.2"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -2,7 +2,10 @@
package teaconst
import "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"os"
)
var (
// 流量统计
@@ -12,6 +15,7 @@ var (
NodeId int64 = 0
NodeIdString = ""
IsDaemon = len(os.Args) > 1 && os.Args[1] == "daemon"
GlobalProductName = nodeconfigs.DefaultProductName

View File

@@ -195,14 +195,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// new connections rate
var newConnectionsRate = tcpConfig.NewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = 100000
// new connections rate (minutely)
var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = 100000
}
}
var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
if newConnectionsMinutelyRateBlockTimeout < 0 {
newConnectionsMinutelyRateBlockTimeout = 0
}
// new connections rate (secondly)
var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = 10000
}
}
var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
if newConnectionsSecondlyRateBlockTimeout < 0 {
newConnectionsSecondlyRateBlockTimeout = 0
}
// 检查是否有变化
var hasChanges = false
@@ -215,7 +232,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
hasChanges = true
break
}
@@ -251,6 +272,7 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// TODO 让用户选择是drop还是reject
if maxConnectionsPerIP > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
var stderr = &bytes.Buffer{}
@@ -261,14 +283,47 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
if newConnectionsRate > 0 {
// TODO 思考是否有惩罚机制
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
// 超过一定速率就drop或者加入黑名单分钟
// TODO 让用户选择是drop还是reject
if newConnectionsMinutelyRate > 0 {
if newConnectionsMinutelyRateBlockTimeout > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
} else {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
// 超过一定速率就drop或者加入黑名单
// TODO 让用户选择是drop还是reject
if newConnectionsSecondlyRate > 0 {
if newConnectionsSecondlyRateBlockTimeout > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
} else {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
}
@@ -335,14 +390,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
for _, rule := range rules {
var pieces = this.decodeUserData(rule.UserData())
if len(pieces) != 4 {
if len(pieces) < 4 {
continue
}
if pieces[0] != "tcp" {
continue
}
switch pieces[2] {
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
err := chain.DeleteRule(rule)
if err != nil {
return err

View File

@@ -0,0 +1,47 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package firewalls
import (
"github.com/iwind/TeaGo/types"
"strings"
"sync"
"time"
)
type BaseFirewall struct {
locker sync.Mutex
latestIPTimes []string // [ip@time, ....]
}
// 检查是否在最近添加过
func (this *BaseFirewall) checkLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -3,6 +3,8 @@
package firewalls
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
@@ -10,15 +12,22 @@ import (
"strings"
)
type firewalldCmd struct {
cmd *exec.Cmd
denyIP string
}
type Firewalld struct {
BaseFirewall
isReady bool
exe string
cmdQueue chan *exec.Cmd
cmdQueue chan *firewalldCmd
}
func NewFirewalld() *Firewalld {
var firewalld = &Firewalld{
cmdQueue: make(chan *exec.Cmd, 4096),
cmdQueue: make(chan *firewalldCmd, 4096),
}
path, err := exec.LookPath("firewall-cmd")
@@ -40,13 +49,19 @@ func NewFirewalld() *Firewalld {
func (this *Firewalld) init() {
goman.New(func() {
for cmd := range this.cmdQueue {
for c := range this.cmdQueue {
var cmd = c.cmd
err := cmd.Run()
if err != nil {
if strings.HasPrefix(err.Error(), "Warning:") {
continue
}
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
} else {
// 关闭连接
if len(c.denyIP) > 0 {
conns.SharedMap.CloseIPConns(c.denyIP)
}
}
}
})
@@ -71,7 +86,7 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
return nil
}
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
return nil
}
@@ -81,12 +96,12 @@ func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol
{
var cmd = exec.Command(this.exe, "--add-port="+port, "--permanent")
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
{
var cmd = exec.Command(this.exe, "--add-port="+port)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
}
@@ -98,7 +113,7 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
return nil
}
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
return nil
}
@@ -107,12 +122,12 @@ func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol str
{
var cmd = exec.Command(this.exe, "--remove-port="+port, "--permanent")
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
{
var cmd = exec.Command(this.exe, "--remove-port="+port)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
return nil
@@ -130,6 +145,12 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -139,14 +160,20 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
this.pushCmd(cmd, ip)
return nil
}
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -156,7 +183,18 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
if async {
this.pushCmd(cmd, ip)
return nil
}
// 关闭连接
defer conns.SharedMap.CloseIPConns(ip)
err := cmd.Run()
if err != nil {
return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
}
return nil
}
@@ -164,6 +202,7 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -171,14 +210,14 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
for _, action := range []string{"reject", "drop"} {
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
return nil
}
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
func (this *Firewalld) pushCmd(cmd *exec.Cmd, denyIP string) {
select {
case this.cmdQueue <- cmd:
case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
default:
// we discard the command
}

View File

@@ -23,7 +23,10 @@ type FirewallInterface interface {
RejectSourceIP(ip string, timeoutSeconds int) error
// DropSourceIP 丢弃某个源IP数据
DropSourceIP(ip string, timeoutSeconds int) error
// ip 要封禁的IP
// timeoutSeconds 过期时间
// async 是否异步
DropSourceIP(ip string, timeoutSeconds int, async bool) error
// RemoveSourceIP 删除某个源IP
RemoveSourceIP(ip string) error

View File

@@ -47,7 +47,7 @@ func (this *MockFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
}
// DropSourceIP 丢弃某个源IP数据
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
_ = ip
_ = timeoutSeconds
return nil

View File

@@ -7,8 +7,11 @@ package firewalls
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
"net"
@@ -21,9 +24,13 @@ import (
// check nft status, if being enabled we load it automatically
func init() {
if teaconst.IsDaemon {
return
}
if runtime.GOOS == "linux" {
var ticker = time.NewTicker(3 * time.Minute)
go func() {
goman.New(func() {
for range ticker.C {
// if already ready, we break
if nftablesIsReady {
@@ -48,7 +55,7 @@ func init() {
break
}
}
}()
})
}
}
@@ -74,9 +81,16 @@ func (this *nftablesTableDefinition) protocol() string {
return "ip"
}
type blockIPItem struct {
action string
ip string
timeoutSeconds int
}
func NewNFTablesFirewall() (*NFTablesFirewall, error) {
var firewall = &NFTablesFirewall{
conn: nftables.NewConn(),
conn: nftables.NewConn(),
dropIPQueue: make(chan *blockIPItem, 4096),
}
err := firewall.init()
if err != nil {
@@ -87,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
}
type NFTablesFirewall struct {
BaseFirewall
conn *nftables.Conn
isReady bool
version string
@@ -98,6 +114,8 @@ type NFTablesFirewall struct {
denyIPv6Set *nftables.Set
firewalld *Firewalld
dropIPQueue chan *blockIPItem
}
func (this *NFTablesFirewall) init() error {
@@ -243,6 +261,18 @@ func (this *NFTablesFirewall) init() error {
nftablesIsReady = true
nftablesInstance = this
goman.New(func() {
for ipItem := range this.dropIPQueue {
switch ipItem.action {
case "drop":
err = this.DropSourceIP(ipItem.ip, ipItem.timeoutSeconds, false)
if err != nil {
remotelogs.Warn("NFTABLES", "drop ip '"+ipItem.ip+"' failed: "+err.Error())
}
}
}
})
// load firewalld
var firewalld = NewFirewalld()
if firewalld.IsReady() {
@@ -307,16 +337,40 @@ func (this *NFTablesFirewall) AllowSourceIP(ip string) error {
// RejectSourceIP 拒绝某个源IP连接
// we did not create set for drop ip, so we reuse DropSourceIP() method here
func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
return this.DropSourceIP(ip, timeoutSeconds)
return this.DropSourceIP(ip, timeoutSeconds, true)
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
var data = net.ParseIP(ip)
if data == nil {
return errors.New("invalid ip '" + ip + "'")
}
// 尝试关闭连接
conns.SharedMap.CloseIPConns(ip)
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
if async {
select {
case this.dropIPQueue <- &blockIPItem{
action: "drop",
ip: ip,
timeoutSeconds: timeoutSeconds,
}:
default:
return errors.New("drop ip queue is full")
}
return nil
}
// 再次尝试关闭连接
defer conns.SharedMap.CloseIPConns(ip)
if strings.Contains(ip, ":") { // ipv6
if this.denyIPv6Set == nil {
return errors.New("ipv6 ip set is nil")
@@ -393,3 +447,35 @@ func (this *NFTablesFirewall) readVersion(nftPath string) string {
}
return versionMatches[1]
}
// 检查是否在最近添加过
func (this *NFTablesFirewall) existLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -51,7 +51,7 @@ func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) erro
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
return nil
}

View File

@@ -3,6 +3,7 @@
package goman
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"runtime"
"sync"
"time"
@@ -14,6 +15,10 @@ var instanceId = uint64(0)
// New 新创建goroutine
func New(f func()) {
if teaconst.IsDaemon {
return
}
_, file, line, _ := runtime.Caller(1)
go func() {
@@ -42,6 +47,10 @@ func New(f func()) {
// NewWithArgs 创建带有参数的goroutine
func NewWithArgs(f func(args ...interface{}), args ...interface{}) {
if teaconst.IsDaemon {
return
}
_, file, line, _ := runtime.Caller(1)
go func() {

View File

@@ -1,130 +0,0 @@
// 源码改自https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
package iplibrary
import (
"errors"
"io/ioutil"
"strconv"
"strings"
)
const (
IndexBlockLength = 12
)
var err error
type IP2Region struct {
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
lastIndexPtr int64
totalBlocks int64
dbData []byte
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
func getIpInfo(cityId int64, line []byte) *IpInfo {
lineSlice := strings.Split(string(line), "|")
ipInfo := &IpInfo{}
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
ipInfo.Country = lineSlice[0]
ipInfo.Region = lineSlice[1]
ipInfo.Province = lineSlice[2]
ipInfo.City = lineSlice[3]
ipInfo.ISP = lineSlice[4]
return ipInfo
}
func NewIP2Region(path string) (*IP2Region, error) {
var region = &IP2Region{}
region.dbData, err = ioutil.ReadFile(path)
if err != nil {
return nil, err
}
region.firstIndexPtr = region.ipLongAtOffset(0)
region.lastIndexPtr = region.ipLongAtOffset(4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
return region, nil
}
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
ip, err := ip2long(ipStr)
if err != nil {
return nil, err
}
h := this.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m*IndexBlockLength
sip := this.ipLongAtOffset(p)
if ip < sip {
h = m - 1
} else {
eip := this.ipLongAtOffset(p + 4)
if ip > eip {
l = m + 1
} else {
dataPtr = this.ipLongAtOffset(p + 8)
break
}
}
}
if dataPtr == 0 {
return nil, nil
}
dataLen := (dataPtr >> 24) & 0xFF
dataPtr = dataPtr & 0x00FFFFFF
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
}
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
return int64(this.dbData[offset]) |
int64(this.dbData[offset+1])<<8 |
int64(this.dbData[offset+2])<<16 |
int64(this.dbData[offset+3])<<24
}
func ip2long(IpStr string) (int64, error) {
bits := strings.Split(IpStr, ".")
if len(bits) != 4 {
return 0, errors.New("ip format error")
}
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24-8*i)
}
return sum, nil
}

View File

@@ -18,7 +18,8 @@ import (
type IPListDB struct {
db *sql.DB
itemTableName string
itemTableName string
deleteExpiredItemsStmt *sql.Stmt
deleteItemStmt *sql.Stmt
insertItemStmt *sql.Stmt
@@ -53,7 +54,11 @@ func (this *IPListDB) init() error {
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = this.dir + "/ip_list.db"
_ = os.Remove(path + "-shm")
_ = os.Remove(path + "-wal")
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
@@ -194,12 +199,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
return 0
}
row := this.selectMaxVersionStmt.QueryRow()
var row = this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err = row.Scan(&version)
err := row.Scan(&version)
if err != nil {
return 0
}

View File

@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.AddItem(&pb.IPItem{
Id: 1,
IpFrom: "192.168.1.101",
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.Close()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -1,13 +0,0 @@
package iplibrary
type LibraryInterface interface {
// Load 加载数据库文件
Load(dbPath string) error
// Lookup 查询IP
// 返回结果有可能为空
Lookup(ip string) (*Result, error)
// Close 关闭数据库文件
Close()
}

View File

@@ -1,83 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"net"
"strings"
)
type IP2RegionLibrary struct {
db *IP2Region
}
func (this *IP2RegionLibrary) Load(dbPath string) error {
db, err := NewIP2Region(dbPath)
if err != nil {
return err
}
this.db = db
return nil
}
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
// 暂不支持IPv6
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
if this.db == nil {
return nil, errors.New("library has not been loaded")
}
defer func() {
// 防止panic发生
err := recover()
if err != nil {
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
}
}()
info, err := this.db.MemorySearch(ip)
if err != nil {
return nil, err
}
if info == nil {
return nil, nil
}
if info.Country == "0" {
info.Country = ""
}
if info.Region == "0" {
info.Region = ""
}
if info.Province == "0" {
info.Province = ""
}
if info.City == "0" {
info.City = ""
}
if info.ISP == "0" {
info.ISP = ""
}
return &Result{
CityId: info.CityId,
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
ISP: info.ISP,
}, nil
}
func (this *IP2RegionLibrary) Close() {
}

View File

@@ -1,113 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
"sync"
"testing"
"time"
)
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
var mem = &runtime.MemStats{}
runtime.ReadMemStats(mem)
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var mem2 = &runtime.MemStats{}
runtime.ReadMemStats(mem2)
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
}
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"8.8.9.9"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var count = 4000
var wg = sync.WaitGroup{}
wg.Add(count)
for i := 0; i < count; i++ {
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
}()
}
wg.Done()
t.Log("ok")
}
func TestIP2RegionLibrary_Memory(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 1_000_000; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
var library = &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_, _ = library.Lookup("8.8.8.8")
}
}

View File

@@ -1,95 +0,0 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/files"
"github.com/iwind/TeaGo/types"
"regexp"
"strings"
)
var SharedManager = NewManager()
var SharedLibrary LibraryInterface
func init() {
events.On(events.EventLoaded, func() {
// 初始化
library, err := SharedManager.Load()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
return
}
SharedLibrary = library
})
}
type Manager struct {
code string
}
func NewManager() *Manager {
return &Manager{}
}
func (this *Manager) Load() (LibraryInterface, error) {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return nil, err
}
config := nodeConfig.GlobalConfig
if config == nil {
config = &serverconfigs.GlobalConfig{}
}
// 当前正在使用的IP库代号
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
dir := Tea.Root + "/resources/ipdata/" + code
var lastVersion int64 = -1
lastFilename := ""
for _, file := range files.NewFile(dir).List() {
filename := file.Name()
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
if reg.MatchString(filename) { // 先查找有版本号的
result := reg.FindStringSubmatch(filename)
version := types.Int64(result[1])
if version > lastVersion {
lastVersion = version
lastFilename = filename
}
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
if lastVersion == -1 {
lastFilename = filename
lastVersion = 0
}
}
}
if len(lastFilename) == 0 {
return nil, errors.New("ip library file not found")
}
var libraryPtr LibraryInterface
switch code {
case serverconfigs.IPLibraryTypeIP2Region:
libraryPtr = &IP2RegionLibrary{}
default:
return nil, errors.New("invalid ip library code '" + code + "'")
}
err = libraryPtr.Load(dir + "/" + lastFilename)
if err != nil {
return nil, err
}
return libraryPtr, nil
}

View File

@@ -1,156 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCityManager = NewCityManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCityManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCityManager.Stop()
})
}
// CityManager 中国省份信息管理
type CityManager struct {
ticker *time.Ticker
cacheFile string
cityMap map[string]int64 // provinceName_cityName => cityName
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCityManager() *CityManager {
return &CityManager{
cacheFile: Tea.Root + "/configs/region_city.json.cache",
cityMap: map[string]int64{},
}
}
func (this *CityManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("City_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
}
}
func (this *CityManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
this.locker.RLock()
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *CityManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.cityMap = m
}
return nil
}
// 更新城市信息
func (this *CityManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCityRPC().FindAllEnabledRegionCities(rpcClient.Context(), &pb.FindAllEnabledRegionCitiesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, city := range resp.RegionCities {
for _, code := range city.Codes {
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.cityMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,14 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewCityManager(t *testing.T) {
var manager = NewCityManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup(16, "许昌市"))
}

View File

@@ -1,154 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCountryManager = NewCountryManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCountryManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCountryManager.Stop()
})
}
// CountryManager 国家/地区信息管理
type CountryManager struct {
ticker *time.Ticker
cacheFile string
countryMap map[string]int64 // countryName => countryId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCountryManager() *CountryManager {
return &CountryManager{
cacheFile: Tea.Root + "/configs/region_country.json.cache",
countryMap: map[string]int64{},
}
}
func (this *CountryManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
}
}
func (this *CountryManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
this.locker.RLock()
countryId, _ = this.countryMap[countryName]
this.locker.RUnlock()
return countryId
}
// 从缓存中读取
func (this *CountryManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.countryMap = m
}
return nil
}
// 更新国家信息
func (this *CountryManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCountryRPC().FindAllEnabledRegionCountries(rpcClient.Context(), &pb.FindAllEnabledRegionCountriesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, country := range resp.RegionCountries {
for _, code := range country.Codes {
m[code] = country.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.countryMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestCountryManager_load(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop(t *testing.T) {
manager := NewCountryManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop_skip(t *testing.T) {
manager := NewCountryManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestCountryManager_Lookup(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
}
func BenchmarkCountryManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewCountryManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("中国")
}
}

View File

@@ -43,7 +43,7 @@ type IPListManager struct {
func NewIPListManager() *IPListManager {
return &IPListManager{
pageSize: 500,
pageSize: 1000,
listMap: map[int64]*IPList{},
}
}
@@ -111,15 +111,19 @@ func (this *IPListManager) init() {
var size int64 = 1000
for {
items, err := db.ReadItems(offset, size)
var l = len(items)
if err != nil {
remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
} else {
if len(items) == 0 {
if l == 0 {
break
}
this.processItems(items, false)
if int64(l) < size {
break
}
}
offset += int64(len(items))
offset += int64(l)
}
}
}
@@ -144,7 +148,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
if err != nil {
return false, err
}
itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
Version: this.version,
Size: this.pageSize,
})
@@ -155,7 +159,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
}
return false, err
}
items := itemsResp.IpItems
var items = itemsResp.IpItems
if len(items) == 0 {
return false, nil
}
@@ -183,7 +187,6 @@ func (this *IPListManager) FindList(listId int64) *IPList {
}
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
this.locker.Lock()
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
var list *IPList
@@ -203,11 +206,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
list = GlobalWhiteIPList
}
} else { // 其他List
this.locker.Lock()
list = this.listMap[item.ListId]
this.locker.Unlock()
}
if list == nil {
list = NewIPList()
this.locker.Lock()
this.listMap[item.ListId] = list
this.locker.Unlock()
}
changedLists[list] = zero.New()
@@ -246,8 +253,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
changedList.Sort()
}
this.locker.Unlock()
if fromRemote {
var latestVersion = items[len(items)-1].Version
if latestVersion > this.version {

View File

@@ -1,155 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedProviderManager = NewProviderManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProviderManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProviderManager.Stop()
})
}
// ProviderManager 中国省份信息管理
type ProviderManager struct {
ticker *time.Ticker
cacheFile string
providerMap map[string]int64 // name => id
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProviderManager() *ProviderManager {
return &ProviderManager{
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
providerMap: map[string]int64{},
}
}
func (this *ProviderManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
}
}
func (this *ProviderManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
this.locker.RLock()
providerId, _ = this.providerMap[providerName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *ProviderManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.providerMap = m
}
return nil
}
// 更新服务商信息
func (this *ProviderManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProviderRPC().FindAllEnabledRegionProviders(rpcClient.Context(), &pb.FindAllEnabledRegionProvidersRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, provider := range resp.RegionProviders {
for _, code := range provider.Codes {
m[code] = provider.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.providerMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,15 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewProviderManager(t *testing.T) {
var manager = NewProviderManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("阿里云"))
t.Log(manager.Lookup("阿里云2"))
}

View File

@@ -1,161 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
const (
ChinaCountryId int64 = 1
)
var SharedProvinceManager = NewProvinceManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProvinceManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProvinceManager.Stop()
})
}
// ProvinceManager 中国省份信息管理
type ProvinceManager struct {
ticker *time.Ticker
cacheFile string
provinceMap map[string]int64 // provinceName => provinceId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProvinceManager() *ProvinceManager {
return &ProvinceManager{
cacheFile: Tea.Root + "/configs/region_province.json.cache",
provinceMap: map[string]int64{},
}
}
func (this *ProvinceManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
}
}
func (this *ProvinceManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
this.locker.RLock()
provinceId, _ = this.provinceMap[provinceName]
this.locker.RUnlock()
return provinceId
}
// 从缓存中读取
func (this *ProvinceManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.provinceMap = m
}
return nil
}
// 更新省份信息
func (this *ProvinceManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProvinceRPC().FindAllEnabledRegionProvincesWithCountryId(rpcClient.Context(), &pb.FindAllEnabledRegionProvincesWithCountryIdRequest{
RegionCountryId: ChinaCountryId,
})
if err != nil {
return err
}
m := map[string]int64{}
for _, province := range resp.RegionProvinces {
for _, code := range province.Codes {
m[code] = province.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.provinceMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestProvinceManager_load(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop(t *testing.T) {
manager := NewProvinceManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop_skip(t *testing.T) {
manager := NewProvinceManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestProvinceManager_Lookup(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
}
func BenchmarkProvinceManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewProvinceManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("安徽省")
}
}

View File

@@ -1,26 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestManager_Load(t *testing.T) {
dbs.NotifyReady()
manager := NewManager()
lib, err := manager.Load()
if err != nil {
t.Fatal(err)
}
t.Log(lib.Lookup("1.2.3.4"))
t.Log(lib.Lookup("2.3.4.5"))
t.Log(lib.Lookup("200.200.200.200"))
t.Log(lib.Lookup("202.106.0.20"))
}
func TestNewManager(t *testing.T) {
dbs.NotifyReady()
t.Log(SharedLibrary)
}

View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"os"
"time"
)
var SharedUpdater = NewUpdater()
func init() {
events.On(events.EventStart, func() {
goman.New(func() {
SharedUpdater.Start()
})
})
events.On(events.EventQuit, func() {
SharedUpdater.Stop()
})
}
// Updater IP库更新程序
type Updater struct {
ticker *time.Ticker
}
// NewUpdater 获取新对象
func NewUpdater() *Updater {
return &Updater{}
}
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新因为通常不需要更新IP库
this.ticker = time.NewTicker(1 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
}
}
func (this *Updater) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
// 单次任务
func (this *Updater) loop() error {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return err
}
if nodeConfig.GlobalConfig == nil {
return nil
}
code := nodeConfig.GlobalConfig.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
if err != nil {
return err
}
lib := libraryResp.IpLibrary
if lib == nil || lib.File == nil {
return nil
}
typeInfo := serverconfigs.FindIPLibraryWithType(code)
if typeInfo == nil {
return errors.New("invalid ip library code '" + code + "'")
}
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
// 是否已经存在
_, err = os.Stat(path)
if err == nil {
return nil
}
// 开始下载
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
if err != nil {
return err
}
chunkIds := fileChunkIdsResp.FileChunkIds
if len(chunkIds) == 0 {
return nil
}
isOk := false
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer func() {
// 如果保存不成功就直接删除
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
for _, chunkId := range chunkIds {
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
chunk := chunkResp.FileChunk
if chunk == nil {
continue
}
_, err = fp.Write(chunk.Data)
if err != nil {
return err
}
}
err = fp.Close()
if err != nil {
return err
}
// 重新加载
library, err := SharedManager.Load()
if err != nil {
return err
}
SharedLibrary = library
isOk = true
return nil
}

View File

@@ -1,18 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestUpdater_loop(t *testing.T) {
dbs.NotifyReady()
updater := NewUpdater()
err := updater.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -4,6 +4,7 @@ package metrics
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"strconv"
"sync"
@@ -11,7 +12,15 @@ import (
var SharedManager = NewManager()
func init() {
events.On(events.EventQuit, func() {
SharedManager.Quit()
})
}
type Manager struct {
isQuiting bool
tasks map[int64]*Task // itemId => *Task
categoryTasks map[string][]*Task // category => []*Task
locker sync.RWMutex
@@ -29,6 +38,10 @@ func NewManager() *Manager {
}
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
if this.isQuiting {
return
}
this.locker.Lock()
defer this.locker.Unlock()
@@ -101,6 +114,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
// Add 添加数据
func (this *Manager) Add(obj MetricInterface) {
if this.isQuiting {
return
}
this.locker.RLock()
for _, task := range this.categoryTasks[obj.MetricCategory()] {
task.Add(obj)
@@ -119,3 +136,17 @@ func (this *Manager) HasTCPMetrics() bool {
func (this *Manager) HasUDPMetrics() bool {
return this.hasUDPMetrics
}
// Quit 退出管理器
func (this *Manager) Quit() {
this.isQuiting = true
remotelogs.Println("METRIC_MANAGER", "quit")
this.locker.Lock()
for _, task := range this.tasks {
_ = task.Stop()
}
this.tasks = map[int64]*Task{}
this.locker.Unlock()
}

View File

@@ -16,6 +16,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
@@ -23,18 +24,19 @@ import (
"time"
)
const MaxQueueSize = 2048
const MaxQueueSize = 256 // TODO 可以配置,可以在单个任务里配置
// Task 单个指标任务
// 数据库存储:
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 上传每个服务中排序最前面的 N 个数据
//
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 只上传每个服务中排序最前面的 N 个数据
type Task struct {
item *serverconfigs.MetricItemConfig
isLoaded bool
@@ -88,7 +90,11 @@ func (this *Task) Init() error {
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
_ = os.Remove(path + "-shm")
_ = os.Remove(path + "-wal")
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
@@ -372,7 +378,9 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
for _, serverId := range serverIds {
for _, currentTime := range times {
idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
var t = trackers.Begin("[METRIC]SELECT_TOP_STMT")
rows, err := this.selectTopStmt.Query(serverId, this.item.Version, currentTime)
t.End()
if err != nil {
return nil, err
}
@@ -421,7 +429,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
return nil, err
}
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
MetricStats: pbStats,
Time: currentTime,
ServerId: serverId,

View File

@@ -69,7 +69,7 @@ func (this *ValueQueue) Loop() error {
}
for value := range this.valuesChan {
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
Item: value.Item,
ValueJSON: value.ValueJSON,
CreatedAt: value.CreatedAt,

View File

@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
if err != nil {
statusErr, ok:= status.FromError(err)
statusErr, ok := status.FromError(err)
if ok {
logs.Println(statusErr.Code())
}

View File

@@ -73,7 +73,7 @@ func (this *APIStream) loop() error {
cancelFunc()
}()
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
nodeStream, err := rpcClient.NodeRPC.NodeStream(ctx)
if err != nil {
if this.isQuiting {
return nil

View File

@@ -5,12 +5,14 @@ package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
"net"
"os"
@@ -22,6 +24,8 @@ import (
// ClientConn 客户端连接
type ClientConn struct {
BaseClientConn
once sync.Once
isTLS bool
@@ -31,29 +35,28 @@ type ClientConn struct {
isLO bool // 是否为环路
hasResetSYNFlood bool
BaseClientConn
}
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
if quickClose {
// TCP
tcpConn, ok := conn.(*net.TCPConn)
if ok {
// TODO 可以在配置中设置此值
_ = tcpConn.SetLinger(nodeconfigs.DefaultTCPLinger)
}
}
func NewClientConn(rawConn net.Conn, isTLS bool, quickClose bool) net.Conn {
// 是否为环路
var remoteAddr = conn.RemoteAddr().String()
var remoteAddr = rawConn.RemoteAddr().String()
var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
return &ClientConn{
BaseClientConn: BaseClientConn{rawConn: conn},
var conn = &ClientConn{
BaseClientConn: BaseClientConn{rawConn: rawConn},
isTLS: isTLS,
isLO: isLO,
}
if quickClose {
// TODO 可以在配置中设置此值
_ = conn.SetLinger(nodeconfigs.DefaultTCPLinger)
}
// 加入到Map
conns.SharedMap.Add(conn)
return conn
}
func (this *ClientConn) Read(b []byte) (n int, err error) {
@@ -110,11 +113,10 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
func (this *ClientConn) Write(b []byte) (n int, err error) {
n, err = this.rawConn.Write(b)
if n > 0 {
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
// 统计当前服务带宽
if this.serverId > 0 {
if !this.isLO { // 环路不统计带宽,避免缓存预热等行为产生带宽
if !this.isLO || Tea.IsTesting() { // 环路不统计带宽,避免缓存预热等行为产生带宽
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
}
}
@@ -132,6 +134,9 @@ func (this *ClientConn) Close() error {
// 不能加条件限制,因为服务配置随时有变化
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
// 从conn map中移除
conns.SharedMap.Remove(this)
return err
}
@@ -177,6 +182,11 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
if timeout <= 0 {
timeout = 600
}
// 关闭当前连接
_ = this.SetLinger(0)
_ = this.Close()
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击当前1分钟"+types.String(result)+"次空连接")
}
}

View File

@@ -2,7 +2,10 @@
package nodes
import "net"
import (
"crypto/tls"
"net"
)
type BaseClientConn struct {
rawConn net.Conn
@@ -42,6 +45,17 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
// SetServerId 设置服务ID
func (this *BaseClientConn) SetServerId(serverId int64) {
this.serverId = serverId
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
nativeConn, ok := conn.NetConn().(ClientConnInterface)
if ok {
nativeConn.SetServerId(serverId)
}
case *ClientConn:
conn.SetServerId(serverId)
}
}
// ServerId 读取当前连接绑定的服务ID
@@ -52,6 +66,17 @@ func (this *BaseClientConn) ServerId() int64 {
// SetUserId 设置所属服务的用户ID
func (this *BaseClientConn) SetUserId(userId int64) {
this.userId = userId
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
nativeConn, ok := conn.NetConn().(ClientConnInterface)
if ok {
nativeConn.SetUserId(userId)
}
case *ClientConn:
conn.SetUserId(userId)
}
}
// UserId 获取当前连接所属服务的用户ID
@@ -66,9 +91,20 @@ func (this *BaseClientConn) RawIP() string {
}
// TCPConn 转换为TCPConn
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
conn, ok := this.rawConn.(*net.TCPConn)
return conn, ok
func (this *BaseClientConn) TCPConn() (tcpConn *net.TCPConn, ok bool) {
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
var internalConn = conn.NetConn()
clientConn, ok := internalConn.(*ClientConn)
if ok {
return clientConn.TCPConn()
}
tcpConn, ok = internalConn.(*net.TCPConn)
default:
tcpConn, ok = this.rawConn.(*net.TCPConn)
}
return
}
// SetLinger 设置Linger

View File

@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"net"
"time"
)
// ClientListener 客户端网络监听
@@ -42,10 +43,28 @@ func (this *ClientListener) Accept() (net.Conn, error) {
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err == nil {
canGoNext, _ := iplibrary.AllowIP(ip, 0)
var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
expiresAt, ok := waf.SharedIPBlackList.ContainsExpires(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if ok {
var timeout = expiresAt - time.Now().Unix()
if timeout > 0 {
canGoNext = false
if !canGoNext || beingDenied {
if timeout > 3600 {
timeout = 3600
}
// 使用本地防火墙延长封禁
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
// 这里 int(int64) 转换的前提是限制了 timeout <= 3600否则将有整型溢出的风险
_ = fw.DropSourceIP(ip, int(timeout), true)
}
}
}
}
if !canGoNext {
tcpConn, ok := conn.(*net.TCPConn)
if ok {
_ = tcpConn.SetLinger(0)
@@ -53,14 +72,6 @@ func (this *ClientListener) Accept() (net.Conn, error) {
_ = conn.Close()
// 使用本地防火墙延长封禁
if beingDenied {
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
_ = fw.DropSourceIP(ip, 60)
}
}
return this.Accept()
}
}

View File

@@ -96,7 +96,7 @@ Loop:
this.rpcClient = client
}
_, err := this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
_, err := this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
if err != nil {
// 是否包含了invalid UTF-8
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
@@ -105,7 +105,7 @@ Loop:
}
// 重新提交
_, err = this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
return err
}

View File

@@ -55,7 +55,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
// logs.PrintAsJSON(accessLog)
//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
accessLog,
}})
if err != nil {
@@ -99,7 +99,7 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
accessLog,
}})
if err != nil {

View File

@@ -15,7 +15,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"io"
"io/ioutil"
"net"
"net/http"
"regexp"
@@ -82,7 +81,7 @@ func (this *HTTPCacheTaskManager) Start() {
if rpcClient != nil {
for taskReq := range this.taskQueue {
_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
_, err := rpcClient.ServerRPC.PurgeServerCache(rpcClient.Context(), taskReq)
if err != nil {
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
}
@@ -105,8 +104,12 @@ func (this *HTTPCacheTaskManager) Loop() error {
return err
}
resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
resp, err := rpcClient.HTTPCacheTaskKeyRPC.FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
if err != nil {
// 忽略连接错误
if rpc.IsConnError(err) {
return nil
}
return err
}
@@ -132,7 +135,7 @@ func (this *HTTPCacheTaskManager) Loop() error {
pbResults = append(pbResults, pbResult)
}
_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
_, err = rpcClient.HTTPCacheTaskKeyRPC.UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
if err != nil {
return err
}
@@ -236,7 +239,7 @@ func (this *HTTPCacheTaskManager) fetchKey(key *pb.HTTPCacheTaskKey) error {
}()
// 读取内容,以便于生成缓存
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.Copy(io.Discard, resp.Body)
// 处理502
if resp.StatusCode == http.StatusBadGateway {

View File

@@ -54,6 +54,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
}
var key = origin.UniqueKey() + "@" + originAddr
var isLnRequest = origin.Id == 0
this.locker.RLock()
client, found := this.clientsMap[key]
@@ -101,6 +102,17 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
idleConns = numberCPU * 8
}
// 可以判断为Ln节点请求
if isLnRequest {
maxConnections *= 8
idleConns *= 8
idleTimeout *= 4
} else if sharedNodeConfig != nil && sharedNodeConfig.Level > 1 {
// Ln节点可以适当增加连接数
maxConnections *= 2
idleConns *= 2
}
// TLS通讯
var tlsConfig = &tls.Config{
InsecureSkipVerify: true,
@@ -149,6 +161,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 5 * time.Second,
TLSClientConfig: tlsConfig,
ReadBufferSize: 8 * 1024,
Proxy: nil,
},
}

View File

@@ -38,7 +38,7 @@ func TestHTTPClientPool_Client(t *testing.T) {
}
func TestHTTPClientPool_cleanClients(t *testing.T) {
origin := &serverconfigs.OriginConfig{
var origin = &serverconfigs.OriginConfig{
Id: 1,
Version: 2,
Addr: &serverconfigs.NetworkAddressConfig{Host: "127.0.0.1", PortRange: "1234"},
@@ -48,8 +48,7 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
t.Fatal(err)
}
pool := NewHTTPClientPool()
pool.clientExpiredDuration = 2 * time.Second
var pool = NewHTTPClientPool()
for i := 0; i < 10; i++ {
t.Log("get", i)

View File

@@ -6,10 +6,10 @@ import (
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -17,7 +17,6 @@ import (
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
@@ -151,7 +150,7 @@ func (this *HTTPRequest) Do() {
// Web配置
err := this.configureWeb(this.ReqServer.Web, true, 0)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to configure the server", "配置服务失败", false)
this.doEnd()
return
}
@@ -284,12 +283,12 @@ func (this *HTTPRequest) doBegin() {
this.web.AccessLogRef.IsOn &&
this.web.AccessLogRef.ContainsField(serverconfigs.HTTPAccessLogFieldRequestBody) {
var err error
this.requestBodyData, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
this.requestBodyData, err = io.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
this.write50x(err, http.StatusBadGateway, "Failed to read request body for access log", "为访问日志读取请求Body失败", false)
return
}
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
this.RawReq.Body = io.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
}
// 跳转
@@ -678,7 +677,7 @@ func (this *HTTPRequest) Format(source string) string {
case "remoteAddrValue":
return this.requestRemoteAddr(false)
case "rawRemoteAddr":
addr := this.RawReq.RemoteAddr
var addr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
@@ -929,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
// geo
if prefix == "geo" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "country.name":
if result != nil {
return result.Country
if result != nil && result.IsOk() {
return result.CountryName()
}
return ""
case "country.id":
if result != nil {
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
if result != nil && result.IsOk() {
return types.String(result.CountryId())
}
return "0"
case "province.name":
if result != nil {
return result.Province
if result != nil && result.IsOk() {
return result.ProvinceName()
}
return ""
case "province.id":
if result != nil {
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
if result != nil && result.IsOk() {
return types.String(result.ProvinceId())
}
return "0"
case "city.name":
if result != nil {
return result.City
if result != nil && result.IsOk() {
return result.CityName()
}
return ""
case "city.id":
if result != nil {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 {
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
} else {
return "0"
}
if result != nil && result.IsOk() {
return types.String(result.CityId())
}
return "0"
case "town.name":
if result != nil && result.IsOk() {
return result.TownName()
}
return ""
case "town.id":
if result != nil && result.IsOk() {
return types.String(result.TownId())
}
return "0"
}
@@ -972,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
// ips
if prefix == "isp" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "name":
if result != nil {
return result.ISP
if result != nil && result.IsOk() {
return result.ProviderName()
}
case "id":
if result != nil {
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
if result != nil && result.IsOk() {
return types.String(result.ProviderId())
}
return "0"
}
@@ -1099,7 +1103,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
}
// Remote-Addr
remoteAddr := this.RawReq.RemoteAddr
var remoteAddr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
if supportVar {
@@ -1168,7 +1172,7 @@ func (this *HTTPRequest) requestRemoteUser() string {
// Path 请求的URL中路径部分
func (this *HTTPRequest) Path() string {
uri, err := url.ParseRequestURI(this.rawURI)
uri, err := url.ParseRequestURI(this.uri)
if err != nil {
return ""
}
@@ -1316,7 +1320,7 @@ func (this *HTTPRequest) RemoteAddr() string {
}
func (this *HTTPRequest) RawRemoteAddr() string {
addr := this.RawReq.RemoteAddr
var addr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
@@ -1390,12 +1394,12 @@ func (this *HTTPRequest) Cookie(name string) string {
return c.Value
}
// DeleteHeader 删除Header
// DeleteHeader 删除请求Header
func (this *HTTPRequest) DeleteHeader(name string) {
this.RawReq.Header.Del(name)
}
// SetHeader 设置Header
// SetHeader 设置请求Header
func (this *HTTPRequest) SetHeader(name string, values []string) {
this.RawReq.Header[name] = values
}
@@ -1424,11 +1428,16 @@ func (this *HTTPRequest) Done() {
func (this *HTTPRequest) Close() {
this.Done()
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
if requestConn == nil {
return
}
lingerConn, ok := requestConn.(LingerConn)
if ok {
_ = lingerConn.SetLinger(0)
}
conn, ok := requestConn.(net.Conn)
if ok {
_ = conn.Close()
@@ -1712,12 +1721,17 @@ func (this *HTTPRequest) canIgnore(err error) bool {
}
// 客户端主动取消
if err == errWritingToClient || err == context.Canceled || err == io.ErrShortWrite || strings.Contains(err.Error(), "write: connection timed out") || strings.Contains(err.Error(), "write: broken pipe") {
if err == errWritingToClient ||
err == context.Canceled ||
err == io.ErrShortWrite ||
strings.Contains(err.Error(), "write: connection") ||
strings.Contains(err.Error(), "write: broken pipe") ||
strings.Contains(err.Error(), "write tcp") {
return true
}
// HTTP/2流错误
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
if err.Error() == "http2: stream closed" || strings.Contains(err.Error(), "stream error") || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
return true
}

View File

@@ -21,7 +21,7 @@ func (this *HTTPRequest) doACME() (shouldStop bool) {
return false
}
keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
keyResp, err := rpcClient.ACMEAuthenticationRPC.FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
if err != nil {
remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
return false

View File

@@ -5,7 +5,7 @@ package nodes
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"io/ioutil"
"io"
"net/http"
)
@@ -19,41 +19,52 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
continue
}
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
if !ref.AuthPolicy.MatchRequest(this.RawReq) {
continue
}
ok, newURI, uriChanged, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
subReq.TLS = this.RawReq.TLS
subReq.RemoteAddr = this.RawReq.RemoteAddr
subReq.Host = this.RawReq.Host
subReq.Proto = this.RawReq.Proto
subReq.ProtoMinor = this.RawReq.ProtoMinor
subReq.ProtoMajor = this.RawReq.ProtoMajor
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
subReq.Body = io.NopCloser(bytes.NewReader([]byte{}))
subReq.Header.Set("Referer", this.URL())
var writer = NewEmptyResponseWriter(this.writer)
this.doSubRequest(writer, subReq)
return writer.StatusCode(), nil
}, this.Format)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
return
}
if b {
if ok {
if uriChanged {
this.uri = newURI
}
this.tags = append(this.tags, ref.AuthPolicy.Type)
return
} else {
// Basic Auth比较特殊
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.ReqHost
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
if ok {
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.ReqHost
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
this.writer.WriteHeader(http.StatusUnauthorized)
this.tags = append(this.tags, ref.AuthPolicy.Type)
return true
}
}

View File

@@ -44,12 +44,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
// 检查服务独立的缓存条件
refType := ""
for _, cacheRef := range this.web.Cache.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
if !cacheRef.IsOn {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
if cacheRef.IsReverse {
return
}
@@ -61,12 +60,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
// 检查策略默认的缓存条件
for _, cacheRef := range cachePolicy.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
if !cacheRef.IsOn {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
if cacheRef.IsReverse {
return
}

View File

@@ -1,32 +1,59 @@
package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"net/http"
"strings"
)
const httpStatusPageTemplate = `<!DOCTYPE html>
<html>
<head>
<title>${status} ${statusMessage}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body>
<h1>${status} ${statusMessage}</h1>
<p>${message}</p>
<address>Request ID: ${requestId}.</address>
</body>
</html>`
func (this *HTTPRequest) write404() {
if this.doPage(http.StatusNotFound) {
this.writeCode(http.StatusNotFound)
}
func (this *HTTPRequest) writeCode(statusCode int) {
if this.doPage(statusCode) {
return
}
this.processResponseHeaders(http.StatusNotFound)
this.writer.WriteHeader(http.StatusNotFound)
_, _ = this.writer.Write([]byte("404 page not found: '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
var pageContent = configutils.ParseVariables(httpStatusPageTemplate, func(varName string) (value string) {
switch varName {
case "status":
return types.String(statusCode)
case "statusMessage":
return http.StatusText(statusCode)
case "requestId":
return this.requestId
case "message":
return "" // 空
}
return "${" + varName + "}"
})
this.processResponseHeaders(statusCode)
this.writer.WriteHeader(statusCode)
_, _ = this.writer.Write([]byte(pageContent))
}
func (this *HTTPRequest) writeCode(code int) {
if this.doPage(code) {
return
}
this.processResponseHeaders(code)
this.writer.WriteHeader(code)
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
}
func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
func (this *HTTPRequest) write50x(err error, statusCode int, enMessage string, zhMessage string, canTryStale bool) {
if err != nil {
this.addError(err)
}
@@ -37,7 +64,7 @@ func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
this.web.Cache.Stale != nil &&
this.web.Cache.Stale.IsOn &&
(len(this.web.Cache.Stale.Status) == 0 || lists.ContainsInt(this.web.Cache.Stale.Status, statusCode)) {
ok := this.doCacheRead(true)
var ok = this.doCacheRead(true)
if ok {
return
}
@@ -47,7 +74,34 @@ func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
if this.doPage(statusCode) {
return
}
// 内置HTML模板
var pageContent = configutils.ParseVariables(httpStatusPageTemplate, func(varName string) (value string) {
switch varName {
case "status":
return types.String(statusCode)
case "statusMessage":
return http.StatusText(statusCode)
case "requestId":
return this.requestId
case "message":
var acceptLanguages = this.RawReq.Header.Get("Accept-Language")
if len(acceptLanguages) > 0 {
var index = strings.Index(acceptLanguages, ",")
if index > 0 {
var firstLanguage = acceptLanguages[:index]
if firstLanguage == "zh-CN" {
return "网站出了一点小问题,原因:" + zhMessage + "。"
}
}
}
return "The site is unavailable now, cause: " + enMessage + "."
}
return "${" + varName + "}"
})
this.processResponseHeaders(statusCode)
this.writer.WriteHeader(statusCode)
_, _ = this.writer.Write([]byte(types.String(statusCode) + " " + http.StatusText(statusCode) + " (Request Id: " + this.requestId + ")"))
_, _ = this.writer.Write([]byte(pageContent))
}

View File

@@ -81,7 +81,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
client, err := fcgi.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to create Fastcgi pool", "Fastcgi池生成失败", false)
return
}
@@ -159,13 +159,13 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
resp, stderr, err := client.Call(fcgiReq)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to read Fastcgi", "读取Fastcgi失败", false)
return
}
if len(stderr) > 0 {
err := errors.New("Fastcgi Error: " + strings.TrimSpace(string(stderr)) + " script: " + maps.NewMap(params).GetString("SCRIPT_FILENAME"))
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to read Fastcgi", "读取Fastcgi失败", false)
return
}

View File

@@ -16,6 +16,6 @@ func (this *HTTPRequest) checkLnRequest() bool {
return false
}
func (this *HTTPRequest) getLnOrigin() *serverconfigs.OriginConfig {
return nil
func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
return nil, 0, false
}

View File

@@ -34,17 +34,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
}
return this.RawReq.ContentLength + hl, true
case "${countConnection}":
metricNewConnMapLocker.Lock()
_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
if ok {
delete(metricNewConnMap, this.RawReq.RemoteAddr)
}
metricNewConnMapLocker.Unlock()
if ok {
return 1, true
} else {
return 0, false
}
return 1, true
}
return 0, false
}

View File

@@ -21,10 +21,32 @@ func (this *HTTPRequest) doReverseProxy() {
return
}
var retries = 3
var failedOriginIds []int64
var failedLnNodeIds []int64
for i := 0; i < retries; i++ {
originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1)
if !shouldRetry {
break
}
if originId > 0 {
failedOriginIds = append(failedOriginIds, originId)
}
if lnNodeId > 0 {
failedLnNodeIds = append(failedLnNodeIds, lnNodeId)
}
}
}
// 请求源站
func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool) (originId int64, lnNodeId int64, shouldRetry bool) {
// 对URL的处理
var stripPrefix = this.reverseProxy.StripPrefix
var requestURI = this.reverseProxy.RequestURI
var requestURIHasVariables = this.reverseProxy.RequestURIHasVariables()
var oldURI = this.uri
var requestHost = ""
if this.reverseProxy.RequestHostType == serverconfigs.RequestHostTypeCustomized {
@@ -41,24 +63,39 @@ func (this *HTTPRequest) doReverseProxy() {
var origin *serverconfigs.OriginConfig
// 二级节点
var hasMultipleLnNodes = false
if this.cacheRef != nil {
origin = this.getLnOrigin()
origin, lnNodeId, hasMultipleLnNodes = this.getLnOrigin(failedLnNodeIds)
if origin != nil {
// 强制变更原来访问的域名
requestHost = this.ReqHost
}
// 回源Header中去除If-None-Match和If-Modified-Since
if !this.cacheRef.EnableIfNoneMatch {
this.DeleteHeader("If-None-Match")
}
if !this.cacheRef.EnableIfModifiedSince {
this.DeleteHeader("If-Modified-Since")
}
}
// 自定义源站
if origin == nil {
origin = this.reverseProxy.NextOrigin(requestCall)
if !isFirstTry {
origin = this.reverseProxy.AnyOrigin(requestCall, failedOriginIds)
}
if origin == nil {
origin = this.reverseProxy.NextOrigin(requestCall)
}
requestCall.CallResponseCallbacks(this.writer)
if origin == nil {
err := errors.New(this.URL() + ": no available origin sites for reverse proxy")
remotelogs.ServerError(this.ReqServer.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "No origin site yet", "尚未配置源站", true)
return
}
originId = origin.Id
if len(origin.StripPrefix) > 0 {
stripPrefix = origin.StripPrefix
@@ -80,7 +117,7 @@ func (this *HTTPRequest) doReverseProxy() {
if origin.Addr == nil {
err := errors.New(this.URL() + ": Origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Origin site did not has a valid address", "源站尚未配置地址", true)
return
}
this.RawReq.URL.Scheme = origin.Addr.Protocol.Primary().Scheme()
@@ -132,7 +169,7 @@ func (this *HTTPRequest) doReverseProxy() {
if originHostIndex < 0 {
var originErr = errors.New(this.URL() + ": Invalid origin address '" + originAddr + "', lacking port")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", originErr.Error())
this.write50x(originErr, http.StatusBadGateway, true)
this.write50x(originErr, http.StatusBadGateway, "No port in origin site address", "源站地址中没有配置端口", true)
return
}
originAddr = originAddr[:originHostIndex+1] + types.String(this.requestServerPort())
@@ -211,16 +248,10 @@ func (this *HTTPRequest) doReverseProxy() {
client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol, this.reverseProxy.FollowRedirects)
if err != nil {
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Create client failed: "+err.Error())
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to create origin site client", "构造源站客户端失败", true)
return
}
// 在HTTP/2下需要防止因为requestBody而导致Content-Length为空的问题
if this.RawReq.ProtoMajor == 2 && this.RawReq.ContentLength == 0 {
_ = this.RawReq.Body.Close()
this.RawReq.Body = nil
}
// 开始请求
resp, err := client.Do(this.RawReq)
if err != nil {
@@ -230,18 +261,35 @@ func (this *HTTPRequest) doReverseProxy() {
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+": Request origin server failed: "+err.Error())
} else if httpErr.Err != context.Canceled {
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
// 是否需要重试
if (originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) && !isLastRetry {
shouldRetry = true
this.uri = oldURI // 恢复备份
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
if httpErr.Err != io.EOF {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
}
return
}
if httpErr.Timeout() {
this.write50x(err, http.StatusGatewayTimeout, true)
this.write50x(err, http.StatusGatewayTimeout, "Read origin site timeout", "源站读取超时", true)
} else if httpErr.Temporary() {
this.write50x(err, http.StatusServiceUnavailable, true)
this.write50x(err, http.StatusServiceUnavailable, "Origin site unavailable now", "源站当前不可用", true)
} else {
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
}
if httpErr.Err != io.EOF {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
@@ -264,7 +312,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
if !isClientError {
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
}
}
if resp != nil && resp.Body != nil {
@@ -410,4 +458,6 @@ func (this *HTTPRequest) doReverseProxy() {
if (err == nil || err == io.EOF) && (closeErr == nil || closeErr == io.EOF) {
this.writer.SetOk()
}
return
}

View File

@@ -114,7 +114,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
return
} else {
this.write50x(err, http.StatusInternalServerError, true)
this.write50x(err, http.StatusInternalServerError, "Failed to stat the file", "查看文件统计信息失败", true)
if !this.canIgnore(err) {
logs.Error(err)
}
@@ -145,7 +145,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
return
} else {
this.write50x(err, http.StatusInternalServerError, true)
this.write50x(err, http.StatusInternalServerError, "Failed to stat the file", "查看文件统计信息失败", true)
if !this.canIgnore(err) {
logs.Error(err)
}
@@ -285,7 +285,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
fileReader, err := os.OpenFile(filePath, os.O_RDONLY, 0444)
if err != nil {
this.write50x(err, http.StatusInternalServerError, true)
this.write50x(err, http.StatusInternalServerError, "Failed to open the file", "试图打开文件失败", true)
return true
}

View File

@@ -35,7 +35,7 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
resp, err := client.Do(req)
if err != nil {
remotelogs.Error("HTTP_REQUEST_URL", req.URL.String()+": "+err.Error())
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to read url", "读取URL失败", false)
return
}
defer func() {

View File

@@ -180,14 +180,14 @@ var httpRequestTimestamp int64
var httpRequestId int32 = 1_000_000
func httpRequestNextId() string {
var unixTime = utils.UnixTimeMilli()
unixTime, unixTimeString := utils.UnixTimeMilliString()
if unixTime > httpRequestTimestamp {
atomic.StoreInt32(&httpRequestId, 1_000_000)
httpRequestTimestamp = unixTime
}
// timestamp + requestId + nodeId
return strconv.FormatInt(unixTime, 10) + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
// timestamp + nodeId + requestId
return unixTimeString + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
}
// 检查是否可以接受某个编码

View File

@@ -2,6 +2,7 @@ package nodes
import (
"bytes"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -11,7 +12,6 @@ import (
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"io"
"io/ioutil"
"net/http"
)
@@ -162,56 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// 检查地区封禁
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
if iplibrary.SharedLibrary != nil {
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
if err != nil {
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
} else if result != nil {
// 检查国家级别封禁
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
var result = iplib.LookupIP(remoteAddr)
if result != nil && result.IsOk() {
// 检查国家/地区级别封禁
var countryId = result.CountryId()
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
// 检查省份封禁
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
return true, false
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 检查省份封禁
var provinceId = result.ProvinceId()
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
return true, false
}
}
}
@@ -359,7 +351,7 @@ func (this *HTTPRequest) WAFSetCacheBody(body []byte) {
// WAFReadBody 读取Body
func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
if this.RawReq.ContentLength > 0 {
data, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, max))
data, err = io.ReadAll(io.LimitReader(this.RawReq.Body, max))
}
return
@@ -368,7 +360,7 @@ func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
// WAFRestoreBody 恢复Body
func (this *HTTPRequest) WAFRestoreBody(data []byte) {
if len(data) > 0 {
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(data), this.RawReq.Body))
this.RawReq.Body = io.NopCloser(io.MultiReader(bytes.NewBuffer(data), this.RawReq.Body))
}
}
@@ -380,6 +372,8 @@ func (this *HTTPRequest) WAFServerId() int64 {
// WAFClose 关闭连接
func (this *HTTPRequest) WAFClose() {
this.Close()
// 这里不要强关IP所有连接避免因为单个服务而影响所有
}
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {

View File

@@ -43,7 +43,7 @@ func (this *HTTPRequest) doWebsocket(requestHost string) {
// TODO 增加N次错误重试重试的时候需要尝试不同的源站
originConn, _, err := OriginConnect(this.origin, this.requestServerPort(), this.RawReq.RemoteAddr, requestHost)
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
this.write50x(err, http.StatusBadGateway, "Failed to connect origin site", "源站连接失败", false)
// 增加失败次数
SharedOriginStateManager.Fail(this.origin, requestHost, this.reverseProxy, func() {
@@ -65,13 +65,13 @@ func (this *HTTPRequest) doWebsocket(requestHost string) {
err = this.RawReq.Write(originConn)
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
this.write50x(err, http.StatusBadGateway, "Failed to write request to origin site", "源站请求初始化失败", false)
return
}
clientConn, _, err := this.writer.Hijack()
if err != nil || clientConn == nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to get origin site connection", "获取源站连接失败", false)
return
}
defer func() {

View File

@@ -26,7 +26,6 @@ import (
_ "image/jpeg"
_ "image/png"
"io"
"io/ioutil"
"net"
"net/http"
"net/textproto"
@@ -524,7 +523,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
this.webpOriginContentType = contentType
this.webpIsEncoding = true
resp.Body = ioutil.NopCloser(&bytes.Buffer{})
resp.Body = io.NopCloser(&bytes.Buffer{})
this.delayRead = true
this.Header().Del("Content-Length")
@@ -1008,30 +1007,35 @@ func (this *HTTPWriter) finishWebP() {
Exact: true,
})
} else if gifImage != nil {
anim := gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
anim.WebPAnimEncoderOptions.SetKmin(9)
anim.WebPAnimEncoderOptions.SetKmax(17)
defer anim.ReleaseMemory()
webpConfig := gowebp.NewWebpConfig()
var webpConfig = gowebp.NewWebpConfig()
//webpConfig.SetLossless(1)
webpConfig.SetQuality(f)
var timeline = 0
var lastErr error
for i, img := range gifImage.Image {
err = anim.AddFrame(img, timeline, webpConfig)
if err != nil {
break
// 有错误直接跳过
lastErr = err
err = nil
}
timeline += gifImage.Delay[i] * 10
}
if err == nil {
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
if lastErr != nil {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
}
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
anim.ReleaseMemory()
}
if err != nil && !this.req.canIgnore(err) {

View File

@@ -0,0 +1,103 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
"io"
"os"
)
type IPLibraryUpdater struct {
}
func NewIPLibraryUpdater() *IPLibraryUpdater {
return &IPLibraryUpdater{}
}
// DataDir 文件目录
func (this *IPLibraryUpdater) DataDir() string {
// data/
var dir = Tea.Root + "/data"
stat, err := os.Stat(dir)
if err == nil && stat.IsDir() {
return dir
}
err = os.Mkdir(dir, 0666)
if err == nil {
return dir
}
remotelogs.Error("IP_LIBRARY_UPDATER", "create directory '"+dir+"' failed: "+err.Error())
// 如果不能创建 data/ 目录,那么使用临时目录
return os.TempDir()
}
// FindLatestFile 检查最新的IP库文件
func (this *IPLibraryUpdater) FindLatestFile() (code string, fileId int64, err error) {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return "", 0, err
}
resp, err := rpcClient.IPLibraryArtifactRPC.FindPublicIPLibraryArtifact(rpcClient.Context(), &pb.FindPublicIPLibraryArtifactRequest{})
if err != nil {
return "", 0, err
}
var artifact = resp.IpLibraryArtifact
if artifact == nil {
return
}
return artifact.Code, artifact.FileId, nil
}
// DownloadFile 下载文件
func (this *IPLibraryUpdater) DownloadFile(fileId int64, writer io.Writer) error {
if fileId <= 0 {
return errors.New("invalid fileId: " + types.String(fileId))
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
chunkIdsResp, err := rpcClient.FileChunkRPC.FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: fileId})
if err != nil {
return err
}
for _, chunkId := range chunkIdsResp.FileChunkIds {
chunkResp, err := rpcClient.FileChunkRPC.DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
var chunk = chunkResp.FileChunk
if chunk == nil {
return errors.New("can not find file chunk with chunk id '" + types.String(chunkId) + "'")
}
_, err = writer.Write(chunk.Data)
if err != nil {
return err
}
}
return nil
}
// LogInfo 普通日志
func (this *IPLibraryUpdater) LogInfo(message string) {
remotelogs.Println("IP_LIBRARY_UPDATER", message)
}
// LogError 错误日志
func (this *IPLibraryUpdater) LogError(err error) {
if err == nil {
return
}
remotelogs.Error("IP_LIBRARY_UPDATER", err.Error())
}

View File

@@ -5,7 +5,6 @@ import (
"crypto/tls"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"golang.org/x/net/http2"
"io"
@@ -13,14 +12,11 @@ import (
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
var httpErrorLogger = log.New(io.Discard, "", 0)
var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool
var metricNewConnMapLocker = &sync.Mutex{}
type contextKey struct {
key string
@@ -55,23 +51,10 @@ func (this *HTTPListener) Serve() error {
switch state {
case http.StateNew:
atomic.AddInt64(&this.countActiveConnections, 1)
// 为指标存储连接信息
if sharedNodeConfig.HasHTTPConnectionMetrics() {
metricNewConnMapLocker.Lock()
metricNewConnMap[conn.RemoteAddr().String()] = zero.New()
metricNewConnMapLocker.Unlock()
}
case http.StateActive, http.StateIdle, http.StateHijacked:
// Nothing to do
case http.StateClosed:
atomic.AddInt64(&this.countActiveConnections, -1)
// 移除指标存储连接信息
// 因为中途配置可能有改变,所以暂时不添加条件
metricNewConnMapLocker.Lock()
delete(metricNewConnMap, conn.RemoteAddr().String())
metricNewConnMapLocker.Unlock()
}
},
ConnContext: func(ctx context.Context, conn net.Conn) context.Context {

View File

@@ -4,6 +4,7 @@ import (
"crypto/tls"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/stats"
@@ -108,8 +109,9 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
// 记录域名排行
tlsConn, ok := conn.(*tls.Conn)
var recordStat = false
var serverName = ""
if ok {
var serverName = tlsConn.ConnectionState().ServerName
serverName = tlsConn.ConnectionState().ServerName
if len(serverName) > 0 {
// 统计
stats.SharedTrafficStatManager.Add(server.Id, serverName, 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
@@ -122,8 +124,9 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
stats.SharedTrafficStatManager.Add(server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
}
originConn, err := this.connectOrigin(server.Id, server.ReverseProxy, conn.RemoteAddr().String())
originConn, err := this.connectOrigin(server.Id, serverName, server.ReverseProxy, conn.RemoteAddr().String())
if err != nil {
_ = conn.Close()
return err
}
@@ -218,37 +221,63 @@ func (this *TCPListener) Close() error {
}
// 连接源站
func (this *TCPListener) connectOrigin(serverId int64, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
func (this *TCPListener) connectOrigin(serverId int64, requestHost string, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
if reverseProxy == nil {
return nil, errors.New("no reverse proxy config")
}
var requestCall = shared.NewRequestCall()
requestCall.Domain = requestHost
var retries = 3
var addr string
var failedOriginIds []int64
for i := 0; i < retries; i++ {
var origin = reverseProxy.NextOrigin(nil)
var origin *serverconfigs.OriginConfig
if len(failedOriginIds) > 0 {
origin = reverseProxy.AnyOrigin(requestCall, failedOriginIds)
}
if origin == nil {
origin = reverseProxy.NextOrigin(requestCall)
}
if origin == nil {
continue
}
// 回源主机名
var requestHost = ""
if len(reverseProxy.RequestHost) > 0 {
requestHost = reverseProxy.RequestHost
}
if len(origin.RequestHost) > 0 {
requestHost = origin.RequestHost
} else if len(reverseProxy.RequestHost) > 0 {
requestHost = reverseProxy.RequestHost
}
conn, addr, err = OriginConnect(origin, this.port, remoteAddr, requestHost)
if err != nil {
failedOriginIds = append(failedOriginIds, origin.Id)
remotelogs.ServerError(serverId, "TCP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
SharedOriginStateManager.Fail(origin, requestHost, reverseProxy, func() {
reverseProxy.ResetScheduling()
})
continue
} else {
if !origin.IsOk {
SharedOriginStateManager.Success(origin, func() {
reverseProxy.ResetScheduling()
})
}
return
}
}
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
if err == nil {
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
}
return
}

View File

@@ -15,6 +15,10 @@ import (
"time"
)
const (
UDPConnLifeSeconds = 30
)
type UDPListener struct {
BaseListener
@@ -135,16 +139,39 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
var retries = 3
var addr string
var failedOriginIds []int64
for i := 0; i < retries; i++ {
var origin = reverseProxy.NextOrigin(nil)
var origin *serverconfigs.OriginConfig
if len(failedOriginIds) > 0 {
origin = reverseProxy.AnyOrigin(nil, failedOriginIds)
}
if origin == nil {
origin = reverseProxy.NextOrigin(nil)
}
if origin == nil {
continue
}
conn, addr, err = OriginConnect(origin, this.port, remoteAddr.String(), "")
if err != nil {
failedOriginIds = append(failedOriginIds, origin.Id)
remotelogs.ServerError(serverId, "UDP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
SharedOriginStateManager.Fail(origin, "", reverseProxy, func() {
reverseProxy.ResetScheduling()
})
continue
} else {
if !origin.IsOk {
SharedOriginStateManager.Success(origin, func() {
reverseProxy.ResetScheduling()
})
}
// PROXY Protocol
if reverseProxy != nil &&
reverseProxy.ProxyProtocol != nil &&
@@ -171,14 +198,17 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
return
}
}
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
if err == nil {
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
}
return
}
// 回收连接
func (this *UDPListener) gcConns() {
this.connLocker.Lock()
closingConns := []*UDPConn{}
var closingConns = []*UDPConn{}
for addr, conn := range this.connMap {
if !conn.IsOk() {
closingConns = append(closingConns, conn)
@@ -203,7 +233,7 @@ type UDPConn struct {
}
func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *net.UDPConn, serverConn *net.UDPConn) *UDPConn {
conn := &UDPConn{
var conn = &UDPConn{
addr: addr,
proxyConn: proxyConn,
serverConn: serverConn,
@@ -217,7 +247,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *ne
}
goman.New(func() {
buffer := utils.BytePool4k.Get()
var buffer = utils.BytePool4k.Get()
defer func() {
utils.BytePool4k.Put(buffer)
}()
@@ -232,9 +262,13 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *ne
break
}
// 记录流量
// 记录流量和带宽
if server != nil {
// 流量
stats.SharedTrafficStatManager.Add(server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
// 带宽
stats.SharedBandwidthStatManager.Add(server.UserId, server.Id, int64(n))
}
}
if err != nil {
@@ -268,5 +302,5 @@ func (this *UDPConn) IsOk() bool {
if !this.isOk {
return false
}
return time.Now().Unix()-this.activatedAt < 30 // 如果超过 N 秒没有活动我们认为是超时
return time.Now().Unix()-this.activatedAt < UDPConnLifeSeconds // 如果超过 N 秒没有活动我们认为是超时
}

View File

@@ -4,12 +4,14 @@ import (
"bytes"
"encoding/json"
"errors"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/configs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
@@ -21,6 +23,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/andybalholm/brotli"
"github.com/iwind/TeaGo/Tea"
@@ -30,7 +33,6 @@ import (
"github.com/iwind/TeaGo/types"
"github.com/iwind/gosock/pkg/gosock"
"gopkg.in/yaml.v3"
"io/ioutil"
"log"
"os"
"os/exec"
@@ -79,7 +81,7 @@ func (this *Node) Test() error {
if err != nil {
return errors.New("test rpc failed: " + err.Error())
}
_, err = rpcClient.APINodeRPC().FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
_, err = rpcClient.APINodeRPC.FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
if err != nil {
return errors.New("test rpc failed: " + err.Error())
}
@@ -102,9 +104,6 @@ func (this *Node) Start() {
// 监听signal
this.listenSignals()
// 启动事件
events.Notify(events.EventStart)
// 本地Sock
err := this.listenSock()
if err != nil {
@@ -112,9 +111,19 @@ func (this *Node) Start() {
return
}
// 启动IP库
remotelogs.Println("NODE", "initializing ip library ...")
err = iplib.InitDefault()
if err != nil {
remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
}
// 检查硬盘类型
this.checkDisk()
// 启动事件
events.Notify(events.EventStart)
// 读取API配置
remotelogs.Println("NODE", "init config ...")
err = this.syncConfig(0)
@@ -147,11 +156,21 @@ func (this *Node) Start() {
// 启动同步计时器
this.startSyncTimer()
// 状态变更计时器
// 更新IP库
goman.New(func() {
iplib.NewUpdater(NewIPLibraryUpdater(), 10*time.Minute).Start()
})
// 监控节点运行状态
goman.New(func() {
NewNodeStatusExecutor().Listen()
})
// 同步时间
goman.New(func() {
clock.Start()
})
// 读取配置
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
@@ -207,8 +226,9 @@ func (this *Node) Start() {
// Daemon 实现守护进程
func (this *Node) Daemon() {
isDebug := lists.ContainsString(os.Args, "debug")
isDebug = true
teaconst.IsDaemon = true
var isDebug = lists.ContainsString(os.Args, "debug")
for {
conn, err := this.sock.Dial()
if err != nil {
@@ -227,13 +247,18 @@ func (this *Node) Daemon() {
_ = os.Setenv("EdgeDaemon", "on")
_ = os.Setenv("EdgeBackground", "on")
cmd := exec.Command(exe)
var cmd = exec.Command(exe)
var buf = &bytes.Buffer{}
cmd.Stderr = buf
err = cmd.Start()
if err != nil {
return err
}
err = cmd.Wait()
if err != nil {
if isDebug {
log.Println("[DAEMON]" + buf.String())
}
return err
}
return nil
@@ -289,17 +314,25 @@ func (this *Node) loop() error {
}
var nodeCtx = rpcClient.Context()
tasksResp, err := rpcClient.NodeTaskRPC().FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
tasksResp, err := rpcClient.NodeTaskRPC.FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
if err != nil {
if rpc.IsConnError(err) && !Tea.IsTesting() {
return nil
}
return errors.New("read node tasks failed: " + err.Error())
}
for _, task := range tasksResp.NodeTasks {
switch task.Type {
case "ipItemChanged":
iplibrary.IPListUpdateNotify <- true
// 防止阻塞
select {
case iplibrary.IPListUpdateNotify <- true:
default:
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -318,13 +351,13 @@ func (this *Node) loop() error {
err = this.syncConfig(task.Version)
}
if err != nil {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: false,
Error: err.Error(),
})
} else {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -333,11 +366,12 @@ func (this *Node) loop() error {
if err != nil {
return err
}
case "nodeVersionChanged":
goman.New(func() {
sharedUpgradeManager.Start()
})
if !sharedUpgradeManager.IsInstalling() {
goman.New(func() {
sharedUpgradeManager.Start()
})
}
case "scriptsChanged":
err = this.reloadCommonScripts()
if err != nil {
@@ -345,7 +379,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -354,7 +388,7 @@ func (this *Node) loop() error {
return err
}
case "nodeLevelChanged":
levelInfoResp, err := rpcClient.NodeRPC().FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
levelInfoResp, err := rpcClient.NodeRPC.FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
if err != nil {
return err
}
@@ -371,7 +405,7 @@ func (this *Node) loop() error {
sharedNodeConfig.ParentNodes = parentNodes
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -380,7 +414,7 @@ func (this *Node) loop() error {
return err
}
case "ddosProtectionChanged":
resp, err := rpcClient.NodeRPC().FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
resp, err := rpcClient.NodeRPC.FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
if err != nil {
return err
}
@@ -407,7 +441,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -452,7 +486,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
nodeCtx := rpcClient.Context()
// TODO 这里考虑只同步版本号有变更的
configResp, err := rpcClient.NodeRPC().FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
configResp, err := rpcClient.NodeRPC.FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
Version: -1, // 更新所有版本
Compress: true,
NodeTaskVersion: taskVersion,
@@ -543,7 +577,7 @@ func (this *Node) syncServerConfig(serverId int64) error {
if err != nil {
return err
}
resp, err := rpcClient.ServerRPC().ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
resp, err := rpcClient.ServerRPC.ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
if err != nil {
return err
}
@@ -605,7 +639,7 @@ func (this *Node) startSyncTimer() {
// 检查集群设置
func (this *Node) checkClusterConfig() error {
configFile := Tea.ConfigFile("cluster.yaml")
data, err := ioutil.ReadFile(configFile)
data, err := os.ReadFile(configFile)
if err != nil {
return err
}
@@ -625,7 +659,7 @@ func (this *Node) checkClusterConfig() error {
}
logs.Println("[NODE]registering node to cluster ...")
resp, err := rpcClient.NodeRPC().RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
resp, err := rpcClient.NodeRPC.RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
if err != nil {
return err
}
@@ -675,7 +709,7 @@ func (this *Node) listenSock() error {
if this.sock.IsListening() {
reply, err := this.sock.Send(&gosock.Command{Code: "pid"})
if err == nil {
return errors.New("error: the process is already running, pid: " + maps.NewMap(reply.Params).GetString("pid"))
return errors.New("error: the process is already running, pid: " + types.String(maps.NewMap(reply.Params).GetInt("pid")))
} else {
return errors.New("error: the process is already running")
}
@@ -713,6 +747,7 @@ func (this *Node) listenSock() error {
_ = this.sock.Close()
events.Notify(events.EventQuit)
events.Notify(events.EventTerminated)
// 监控连接数如果连接数为0则退出进程
goman.New(func() {
@@ -763,20 +798,24 @@ func (this *Node) listenSock() error {
},
})
case "conns":
ipConns, serverConns := sharedClientConnLimiter.Conns()
var addrs = []string{}
var connMap = conns.SharedMap.AllConns()
for _, conn := range connMap {
addrs = append(addrs, conn.RemoteAddr().String())
}
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"ipConns": ipConns,
"serverConns": serverConns,
"total": sharedListenerManager.TotalActiveConnections(),
"addrs": addrs,
"total": len(addrs),
},
})
case "dropIP":
var m = maps.NewMap(cmd.Params)
var ip = m.GetString("ip")
var timeSeconds = m.GetInt("timeoutSeconds")
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds)
var async = m.GetBool("async")
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds, async)
if err != nil {
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
@@ -840,6 +879,11 @@ func (this *Node) listenSock() error {
} else {
_ = cmd.ReplyOk()
}
case "bandwidth":
var m = stats.SharedBandwidthStatManager.Map()
_ = cmd.Reply(&gosock.Command{Params: maps.Map{
"stats": m,
}})
}
})
@@ -850,7 +894,7 @@ func (this *Node) listenSock() error {
})
events.OnKey(events.EventQuit, this, func() {
logs.Println("NODE", "quit unix sock")
remotelogs.Println("NODE", "quit unix sock")
_ = this.sock.Close()
})
@@ -994,19 +1038,22 @@ func (this *Node) reloadServer() {
}
func (this *Node) checkDisk() {
if runtime.GOOS == "linux" {
if runtime.GOOS != "linux" {
return
}
for n := 'a'; n <= 'z'; n++ {
for _, path := range []string{
"/sys/block/vda/queue/rotational",
"/sys/block/sda/queue/rotational",
"/sys/block/vd" + string(n) + "/queue/rotational",
"/sys/block/sd" + string(n) + "/queue/rotational",
} {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
continue
}
if string(data) == "0" {
teaconst.DiskIsFast = true
}
break
return
}
}
}

View File

@@ -9,7 +9,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"os"
"syscall"
)
@@ -25,7 +24,7 @@ func (this *Node) handlePanic() {
var panicFile = Tea.Root + "/logs/panic.log"
// 分析panic
data, err := ioutil.ReadFile(panicFile)
data, err := os.ReadFile(panicFile)
if err == nil {
var index = bytes.Index(data, []byte("panic:"))
if index >= 0 {

View File

@@ -30,10 +30,14 @@ type NodeStatusExecutor struct {
cpuUpdatedTime time.Time
cpuLogicalCount int
cpuPhysicalCount int
ticker *time.Ticker
}
func NewNodeStatusExecutor() *NodeStatusExecutor {
return &NodeStatusExecutor{}
return &NodeStatusExecutor{
ticker: time.NewTicker(30 * time.Second),
}
}
func (this *NodeStatusExecutor) Listen() {
@@ -41,15 +45,12 @@ func (this *NodeStatusExecutor) Listen() {
this.cpuUpdatedTime = time.Now()
this.update()
// TODO 这个时间间隔可以配置
var ticker = time.NewTicker(30 * time.Second)
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("NODE_STATUS", "quit executor")
ticker.Stop()
this.ticker.Stop()
})
for range ticker.C {
for range this.ticker.C {
this.isFirstTime = false
this.update()
}
@@ -124,7 +125,7 @@ func (this *NodeStatusExecutor) update() {
remotelogs.Error("NODE_STATUS", "failed to open rpc: "+err.Error())
return
}
_, err = rpcClient.NodeRPC().UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
_, err = rpcClient.NodeRPC.UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
StatusJSON: jsonData,
})
if err != nil {

View File

@@ -11,7 +11,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/maps"
"io/ioutil"
"os"
"os/exec"
"runtime"
@@ -104,7 +103,7 @@ func (this *SystemServiceManager) setupSystemd(params maps.Map) error {
if output == "enabled" {
// 检查文件路径是否变化
data, err := ioutil.ReadFile("/etc/systemd/system/" + teaconst.SystemdServiceName + ".service")
data, err := os.ReadFile("/etc/systemd/system/" + teaconst.SystemdServiceName + ".service")
if err == nil && bytes.Index(data, []byte(exe)) > 0 {
return nil
}

View File

@@ -60,7 +60,7 @@ func (this *OCSPUpdateTask) Loop() error {
return err
}
resp, err := rpcClient.SSLCertRPC().ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
resp, err := rpcClient.SSLCertRPC.ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
Version: this.version,
Size: 100,
})

View File

@@ -81,7 +81,7 @@ func (this *SyncAPINodesTask) Loop() error {
if err != nil {
return err
}
resp, err := rpcClient.APINodeRPC().FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
resp, err := rpcClient.APINodeRPC.FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
if err != nil {
return err
}

View File

@@ -14,8 +14,10 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
stringutil "github.com/iwind/TeaGo/utils/string"
"github.com/iwind/gosock/pkg/gosock"
"os"
"os/exec"
"path/filepath"
"runtime"
"time"
)
@@ -46,15 +48,12 @@ func (this *UpgradeManager) Start() {
}
this.isInstalling = true
// 还原安装状态
defer func() {
this.isInstalling = false
}()
remotelogs.Println("UPGRADE_MANAGER", "upgrading node ...")
err := this.install()
if err != nil {
remotelogs.Error("UPGRADE_MANAGER", "download failed: "+err.Error())
this.isInstalling = false
return
}
@@ -65,9 +64,16 @@ func (this *UpgradeManager) Start() {
if err != nil {
remotelogs.Error("UPGRADE_MANAGER", err.Error())
}
this.isInstalling = false
})
}
// IsInstalling 检查是否正在安装
func (this *UpgradeManager) IsInstalling() bool {
return this.isInstalling
}
func (this *UpgradeManager) install() error {
// 检查是否有已下载但未安装成功的
if len(this.lastFile) > 0 {
@@ -83,7 +89,7 @@ func (this *UpgradeManager) install() error {
}
// 创建临时文件
dir := Tea.Root + "/tmp"
var dir = Tea.Root + "/tmp"
_, err := os.Stat(dir)
if err != nil {
if os.IsNotExist(err) {
@@ -98,7 +104,7 @@ func (this *UpgradeManager) install() error {
remotelogs.Println("UPGRADE_MANAGER", "downloading new node ...")
path := dir + "/edge-node" + ".tmp"
var path = dir + "/edge-node" + ".tmp"
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0777)
if err != nil {
return err
@@ -120,7 +126,7 @@ func (this *UpgradeManager) install() error {
var sum = ""
var filename = ""
for {
resp, err := client.NodeRPC().DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
resp, err := client.NodeRPC.DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
Os: runtime.GOOS,
Arch: runtime.GOARCH,
ChunkOffset: offset,
@@ -203,16 +209,16 @@ func (this *UpgradeManager) unzip(zipPath string) error {
}
// 先改先前的可执行文件
err := os.Rename(target+"/bin/edge-node", target+"/bin/.edge-node.dist")
hasBackup := err == nil
err := os.Rename(target+"/bin/"+teaconst.ProcessName, target+"/bin/."+teaconst.ProcessName+".dist")
var hasBackup = err == nil
defer func() {
if !isOk && hasBackup {
// 失败时还原
_ = os.Rename(target+"/bin/.edge-node.dist", target+"/bin/edge-node")
_ = os.Rename(target+"/bin/."+teaconst.ProcessName+".dist", target+"/bin/"+teaconst.ProcessName)
}
}()
unzip := utils.NewUnzip(zipPath, target, "edge-node/")
var unzip = utils.NewUnzip(zipPath, target, "edge-node/")
err = unzip.Run()
if err != nil {
return err
@@ -225,6 +231,9 @@ func (this *UpgradeManager) unzip(zipPath string) error {
// 重启
func (this *UpgradeManager) restart() error {
// 关闭当前sock防止无法重启
_ = gosock.NewTmpSock(teaconst.ProcessName).Close()
// 重新启动
if DaemonIsOn && DaemonPid == os.Getppid() {
utils.Exit() // TODO 试着更优雅重启
@@ -241,7 +250,9 @@ func (this *UpgradeManager) restart() error {
events.Notify(events.EventTerminated)
// 启动
cmd := exec.Command(exe, "start")
exe = filepath.Dir(exe) + "/" + teaconst.ProcessName
var cmd = exec.Command(exe, "start")
err = cmd.Start()
if err != nil {
return err

View File

@@ -109,6 +109,8 @@ func TestRegexp_ParseKeywords3(t *testing.T) {
func BenchmarkRegexp_MatchString(b *testing.B) {
var r = re.MustCompile("(?i)(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)(\\s|%09|%0A|(\\+|%20))*(=|%3D)")
b.ResetTimer()
//b.Log("keywords:", r.Keywords())
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
@@ -117,6 +119,8 @@ func BenchmarkRegexp_MatchString(b *testing.B) {
func BenchmarkRegexp_MatchString2(b *testing.B) {
var r = regexp.MustCompile("(?i)(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)(\\s|%09|%0A|(\\+|%20))*(=|%3D)")
b.ResetTimer()
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}

View File

@@ -173,7 +173,6 @@ func ServerSuccess(serverId int64, tag string, description string, logType nodec
}
}
// ServerLog 打印服务相关日志信息
func ServerLog(serverId int64, tag string, description string, logType nodeconfigs.NodeLogType, params maps.Map) {
logs.Println("[" + tag + "]" + description)
@@ -253,6 +252,6 @@ Loop:
return nil
}
_, err = rpcClient.NodeLogRPC().CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
_, err = rpcClient.NodeLogRPC.CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
return err
}

View File

@@ -28,6 +28,27 @@ type RPCClient struct {
conns []*grpc.ClientConn
locker sync.RWMutex
NodeRPC pb.NodeServiceClient
NodeLogRPC pb.NodeLogServiceClient
NodeTaskRPC pb.NodeTaskServiceClient
NodeValueRPC pb.NodeValueServiceClient
HTTPAccessLogRPC pb.HTTPAccessLogServiceClient
HTTPCacheTaskKeyRPC pb.HTTPCacheTaskKeyServiceClient
APINodeRPC pb.APINodeServiceClient
IPLibraryArtifactRPC pb.IPLibraryArtifactServiceClient
IPListRPC pb.IPListServiceClient
IPItemRPC pb.IPItemServiceClient
FileRPC pb.FileServiceClient
FileChunkRPC pb.FileChunkServiceClient
ACMEAuthenticationRPC pb.ACMEAuthenticationServiceClient
ServerRPC pb.ServerServiceClient
ServerDailyStatRPC pb.ServerDailyStatServiceClient
ServerBandwidthStatRPC pb.ServerBandwidthStatServiceClient
MetricStatRPC pb.MetricStatServiceClient
FirewallRPC pb.FirewallServiceClient
SSLCertRPC pb.SSLCertServiceClient
ScriptRPC pb.ScriptServiceClient
}
func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
@@ -35,10 +56,32 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
return nil, errors.New("api config should not be nil")
}
client := &RPCClient{
var client = &RPCClient{
apiConfig: apiConfig,
}
// 初始化RPC实例
client.NodeRPC = pb.NewNodeServiceClient(client)
client.NodeLogRPC = pb.NewNodeLogServiceClient(client)
client.NodeTaskRPC = pb.NewNodeTaskServiceClient(client)
client.NodeValueRPC = pb.NewNodeValueServiceClient(client)
client.HTTPAccessLogRPC = pb.NewHTTPAccessLogServiceClient(client)
client.HTTPCacheTaskKeyRPC = pb.NewHTTPCacheTaskKeyServiceClient(client)
client.APINodeRPC = pb.NewAPINodeServiceClient(client)
client.IPLibraryArtifactRPC = pb.NewIPLibraryArtifactServiceClient(client)
client.IPListRPC = pb.NewIPListServiceClient(client)
client.IPItemRPC = pb.NewIPItemServiceClient(client)
client.FileRPC = pb.NewFileServiceClient(client)
client.FileChunkRPC = pb.NewFileChunkServiceClient(client)
client.ACMEAuthenticationRPC = pb.NewACMEAuthenticationServiceClient(client)
client.ServerRPC = pb.NewServerServiceClient(client)
client.ServerDailyStatRPC = pb.NewServerDailyStatServiceClient(client)
client.ServerBandwidthStatRPC = pb.NewServerBandwidthStatServiceClient(client)
client.MetricStatRPC = pb.NewMetricStatServiceClient(client)
client.FirewallRPC = pb.NewFirewallServiceClient(client)
client.SSLCertRPC = pb.NewSSLCertServiceClient(client)
client.ScriptRPC = pb.NewScriptServiceClient(client)
err := client.init()
if err != nil {
return nil, err
@@ -47,102 +90,10 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
return client, nil
}
func (this *RPCClient) NodeRPC() pb.NodeServiceClient {
return pb.NewNodeServiceClient(this.pickConn())
}
func (this *RPCClient) NodeLogRPC() pb.NodeLogServiceClient {
return pb.NewNodeLogServiceClient(this.pickConn())
}
func (this *RPCClient) NodeTaskRPC() pb.NodeTaskServiceClient {
return pb.NewNodeTaskServiceClient(this.pickConn())
}
func (this *RPCClient) NodeValueRPC() pb.NodeValueServiceClient {
return pb.NewNodeValueServiceClient(this.pickConn())
}
func (this *RPCClient) HTTPAccessLogRPC() pb.HTTPAccessLogServiceClient {
return pb.NewHTTPAccessLogServiceClient(this.pickConn())
}
func (this *RPCClient) HTTPCacheTaskKeyRPC() pb.HTTPCacheTaskKeyServiceClient {
return pb.NewHTTPCacheTaskKeyServiceClient(this.pickConn())
}
func (this *RPCClient) APINodeRPC() pb.APINodeServiceClient {
return pb.NewAPINodeServiceClient(this.pickConn())
}
func (this *RPCClient) IPLibraryRPC() pb.IPLibraryServiceClient {
return pb.NewIPLibraryServiceClient(this.pickConn())
}
func (this *RPCClient) RegionCountryRPC() pb.RegionCountryServiceClient {
return pb.NewRegionCountryServiceClient(this.pickConn())
}
func (this *RPCClient) RegionProvinceRPC() pb.RegionProvinceServiceClient {
return pb.NewRegionProvinceServiceClient(this.pickConn())
}
func (this *RPCClient) RegionCityRPC() pb.RegionCityServiceClient {
return pb.NewRegionCityServiceClient(this.pickConn())
}
func (this *RPCClient) RegionProviderRPC() pb.RegionProviderServiceClient {
return pb.NewRegionProviderServiceClient(this.pickConn())
}
func (this *RPCClient) IPListRPC() pb.IPListServiceClient {
return pb.NewIPListServiceClient(this.pickConn())
}
func (this *RPCClient) IPItemRPC() pb.IPItemServiceClient {
return pb.NewIPItemServiceClient(this.pickConn())
}
func (this *RPCClient) FileRPC() pb.FileServiceClient {
return pb.NewFileServiceClient(this.pickConn())
}
func (this *RPCClient) FileChunkRPC() pb.FileChunkServiceClient {
return pb.NewFileChunkServiceClient(this.pickConn())
}
func (this *RPCClient) ACMEAuthenticationRPC() pb.ACMEAuthenticationServiceClient {
return pb.NewACMEAuthenticationServiceClient(this.pickConn())
}
func (this *RPCClient) ServerRPC() pb.ServerServiceClient {
return pb.NewServerServiceClient(this.pickConn())
}
func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient {
return pb.NewServerDailyStatServiceClient(this.pickConn())
}
func (this *RPCClient) ServerBandwidthStatRPC() pb.ServerBandwidthStatServiceClient {
return pb.NewServerBandwidthStatServiceClient(this.pickConn())
}
func (this *RPCClient) MetricStatRPC() pb.MetricStatServiceClient {
return pb.NewMetricStatServiceClient(this.pickConn())
}
func (this *RPCClient) FirewallRPC() pb.FirewallServiceClient {
return pb.NewFirewallServiceClient(this.pickConn())
}
func (this *RPCClient) SSLCertRPC() pb.SSLCertServiceClient {
return pb.NewSSLCertServiceClient(this.pickConn())
}
// Context 节点上下文信息
func (this *RPCClient) Context() context.Context {
ctx := context.Background()
m := maps.Map{
var ctx = context.Background()
var m = maps.Map{
"timestamp": time.Now().Unix(),
"type": "node",
"userId": 0,
@@ -157,7 +108,7 @@ func (this *RPCClient) Context() context.Context {
utils.PrintError(err)
return context.Background()
}
token := base64.StdEncoding.EncodeToString(data)
var token = base64.StdEncoding.EncodeToString(data)
ctx = metadata.AppendToOutgoingContext(ctx, "nodeId", this.apiConfig.NodeId, "token", token)
return ctx
}
@@ -209,7 +160,7 @@ func (this *RPCClient) UpdateConfig(config *configs.APIConfig) error {
// 初始化
func (this *RPCClient) init() error {
// 重新连接
conns := []*grpc.ClientConn{}
var conns = []*grpc.ClientConn{}
for _, endpoint := range this.apiConfig.RPC.Endpoints {
u, err := url.Parse(endpoint)
if err != nil {
@@ -248,20 +199,24 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
// 检查连接状态
if len(this.conns) > 0 {
availableConns := []*grpc.ClientConn{}
for _, state := range []connectivity.State{connectivity.Ready, connectivity.Idle, connectivity.Connecting} {
var availableConns = []*grpc.ClientConn{}
for _, stateArray := range [][2]connectivity.State{
{connectivity.Ready, connectivity.Idle}, // 优先Ready和Idle
{connectivity.Connecting, connectivity.Connecting},
} {
for _, conn := range this.conns {
if conn.GetState() == state {
var state = conn.GetState()
if state == stateArray[0] || state == stateArray[1] {
availableConns = append(availableConns, conn)
}
}
if len(availableConns) > 0 {
break
return this.randConn(availableConns)
}
}
if len(availableConns) > 0 {
return availableConns[rands.Int(0, len(availableConns)-1)]
return this.randConn(availableConns)
}
// 关闭
@@ -281,5 +236,31 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
return nil
}
return this.conns[rands.Int(0, len(this.conns)-1)]
return this.randConn(this.conns)
}
func (this *RPCClient) Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...grpc.CallOption) error {
var conn = this.pickConn()
if conn == nil {
return errors.New("can not get available grpc connection")
}
return conn.Invoke(ctx, method, args, reply, opts...)
}
func (this *RPCClient) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
var conn = this.pickConn()
if conn == nil {
return nil, errors.New("can not get available grpc connection")
}
return conn.NewStream(ctx, desc, method, opts...)
}
func (this *RPCClient) randConn(conns []*grpc.ClientConn) *grpc.ClientConn {
var l = len(conns)
if l == 0 {
return nil
}
if l == 1 {
return conns[0]
}
return conns[rands.Int(0, l-1)]
}

Some files were not shown because too many files have changed in this diff Show More