Compare commits

...

83 Commits

Author SHA1 Message Date
刘祥超
57cb00edf0 修复无法添加IP到本地防火墙的问题 2022-09-04 17:06:19 +08:00
刘祥超
3fb39b479a 优化IP名单锁 2022-09-03 21:10:13 +08:00
刘祥超
4a1daff143 增加简化的缓存条件设置 2022-09-03 18:16:35 +08:00
刘祥超
dd1dbd424e 修复WAF IP名单测试用例的一个可能异常 2022-09-03 09:56:36 +08:00
刘祥超
305cb4b46e 优化WAF中IP名单 2022-09-03 09:54:25 +08:00
刘祥超
ef90dce29b 优化节点进程退出逻辑 2022-09-02 16:12:58 +08:00
刘祥超
3cb69f4c71 将IP加入黑名单时,同时也会关闭此IP相关的连接 2022-09-02 15:20:58 +08:00
刘祥超
af4cd05df2 增加一处注释 2022-09-01 09:06:21 +08:00
刘祥超
64e0ae80b7 DDoS防护增加秒级连接速率限制 2022-08-31 10:00:38 +08:00
刘祥超
8bba228745 优化代码 2022-08-30 18:49:21 +08:00
刘祥超
8cc06e6707 鉴权成功时也在访问日志中记录鉴权类型 2022-08-30 17:28:58 +08:00
刘祥超
52fdee2eeb 修复访问控制后缓存Key包含认证参数的问题 2022-08-30 12:04:01 +08:00
刘祥超
b5f52dd136 优化鉴权 2022-08-30 11:24:23 +08:00
刘祥超
abda886de5 如果系统安装了ntpdate,则自动尝试利用ntpdate同步时间 2022-08-27 15:28:53 +08:00
刘祥超
18f08525b9 WAF和其他请求关闭连接时更加快速 2022-08-27 10:49:16 +08:00
刘祥超
b2a9a31fe5 增加edge-node bandwidth命令查看实时带宽 2022-08-26 16:47:46 +08:00
刘祥超
f578114aeb 修复HTTPS连接无法记录带宽的问题,优化带宽计算方法 2022-08-26 16:47:42 +08:00
刘祥超
bf4f47fc35 DDoS防护增加单IP TCP新连接速率黑名单 2022-08-26 11:31:43 +08:00
刘祥超
0be951742a WAF优化captcha和js_cookie的失败计数器/增强js_cookie的安全性 2022-08-25 17:06:52 +08:00
刘祥超
59d3d6ae4b WAF标签动作匹配之后可以继续尝试匹配别的分组中的规则集 2022-08-25 16:44:44 +08:00
刘祥超
d061876f7e 增加Javascript Cookie验证 2022-08-25 15:35:32 +08:00
刘祥超
ddaec82415 优化RPC获取服务实例方式 2022-08-24 20:04:46 +08:00
刘祥超
8afd00f00d 删除HTTP/2自动关闭Body的逻辑 2022-08-24 16:43:32 +08:00
刘祥超
0b306f0a22 忽略部分HTTP/2错误提示 2022-08-24 16:30:38 +08:00
刘祥超
b36a36172b 优化代码 2022-08-23 14:53:39 +08:00
刘祥超
770278bbbc 实现IP库检查更新 2022-08-23 14:32:39 +08:00
刘祥超
ca24818571 优化缓存索引内存使用 2022-08-22 09:44:09 +08:00
刘祥超
cd0af22655 优化代码 2022-08-22 08:31:39 +08:00
刘祥超
96cb8d8af7 IP库改为手动初始化 2022-08-21 23:09:47 +08:00
刘祥超
91face15bf 使用新版IP库 2022-08-21 20:37:49 +08:00
刘祥超
6f52df63a5 只有系统内存超过3GB的才缓存Hash 2022-08-20 12:09:15 +08:00
刘祥超
509d81dc66 修复检测是否为高速硬盘只能检查sda/vda设备的问题 2022-08-20 11:59:49 +08:00
刘祥超
817f2a6f91 大幅提升缓存索引性能 2022-08-20 11:47:57 +08:00
刘祥超
d74e10c7a8 缓存任务忽略连接API错误 2022-08-20 08:15:16 +08:00
刘祥超
1a1280b76e 优化代码 2022-08-19 14:50:26 +08:00
刘祥超
85d3b1169f 修复Gif中透明图转换WebP失败的问题 2022-08-19 13:39:46 +08:00
刘祥超
1b32292e0c WebP压缩Gif时,自动跳过转换失败的帧,并只提示最后一次错误 2022-08-19 13:27:18 +08:00
刘祥超
e6ba44fb2f [SQLITE]使用事务批量提交一些缓存相关任务 2022-08-17 21:04:00 +08:00
刘祥超
9b6e022f46 将版本修改为0.5.2 2022-08-17 18:56:48 +08:00
刘祥超
4e9ab19fc0 修复因为fsnotify包引用而无法编译的问题 2022-08-17 12:12:40 +08:00
刘祥超
4d634d8fa5 修复一处HTTPS错误提示 2022-08-15 18:26:23 +08:00
刘祥超
a538282e4f 版本修改为0.5.1 2022-08-15 18:26:10 +08:00
刘祥超
df31921954 优化代码 2022-08-14 16:28:40 +08:00
刘祥超
299abb7b04 缓存本地数据库发生错误时同时提示数据库文件名 2022-08-14 15:17:07 +08:00
刘祥超
85ce63b4d3 优化API命名/优化一处测试用例 2022-08-11 12:44:27 +08:00
刘祥超
9d9ae288bd 优化代码 2022-08-10 14:37:27 +08:00
刘祥超
c0a35eb5e7 优化代码 2022-08-09 22:59:37 +08:00
刘祥超
d1d0ff062b 增加Ln节点向源站的最大连接数 2022-08-08 08:12:49 +08:00
刘祥超
396e8a22c4 改进一处错误提示 2022-08-07 19:02:06 +08:00
刘祥超
4f040db1ef 版本号改为0.5.0 2022-08-07 19:01:45 +08:00
刘祥超
e3cf111344 缓存条件增加If-None-Match和If-Modified-Since是否回源选项;默认不回源,防止源站返回304 2022-08-07 16:37:29 +08:00
刘祥超
28cb3c383d 修复一处注释 2022-08-07 11:18:16 +08:00
刘祥超
10665c0f37 优化代码 2022-08-07 11:12:29 +08:00
刘祥超
4096f11909 修复一处测试用例 2022-08-06 20:52:52 +08:00
刘祥超
5df209b6d5 优化代码 2022-08-04 11:34:06 +08:00
刘祥超
db353fe025 nftables封禁IP使用异步操作 2022-08-04 11:01:16 +08:00
刘祥超
30c3e143b8 edge-node pprof命令增加--addr参数 2022-08-04 10:18:23 +08:00
刘祥超
15fe7b33a4 新增屏蔽错误 2022-08-04 10:04:26 +08:00
刘祥超
17e0666ba4 优化代码 2022-08-03 23:31:08 +08:00
刘祥超
3f24bfaaf5 HTTP读取L2节点时增加最大连接数、最大空闲连接数、最大空闲时间 2022-08-03 20:54:08 +08:00
刘祥超
ceadcfece9 HTTP源站有多个L2节点时发生错误时会主动尝试下一个L2节点 2022-08-03 20:30:59 +08:00
刘祥超
bc706237ef HTTP源站读取错误时自动尝试下一个源站 2022-08-03 19:33:50 +08:00
刘祥超
b8c5a78f2e TCP/TLS/UDP第一次连接源站失败后,自动尝试下一个源站 2022-08-03 16:09:47 +08:00
刘祥超
9ad1c3a3c8 TLS支持默认SNI回源 2022-08-03 11:04:58 +08:00
刘祥超
8cfde43f5d TCP连接源站失败时关闭连接/TCP连接也支持备用源站 2022-08-01 19:28:27 +08:00
刘祥超
6f843b071a 执行IP名单更新任务时防止阻塞/优化节点升级代码 2022-08-01 18:54:54 +08:00
刘祥超
a72b025900 优化忽略客户端关闭连接错误条件 2022-08-01 16:53:08 +08:00
刘祥超
dfb66775d7 40x, 50x提示默认使用HTML;50x提示增加原因信息 2022-07-30 10:48:41 +08:00
刘祥超
c4dca2df30 UDP也记录带宽 2022-07-30 09:30:05 +08:00
刘祥超
a3525bdaa4 修复节点自动升级时无法自动启动的Bug 2022-07-28 14:38:08 +08:00
刘祥超
09390bbb97 优化代码 2022-07-27 18:10:43 +08:00
刘祥超
70ae4391d7 减少Daemon使用的内存 2022-07-26 09:41:43 +08:00
刘祥超
85931f55e1 修改版本号为0.4.11 2022-07-26 09:33:38 +08:00
刘祥超
0f5f03c9ed 取消IO保护 2022-07-26 08:29:22 +08:00
刘祥超
1181a0585b 优化IP名单相关代码/关闭、重启进程时自动关闭IP名单本地缓存数据库 2022-07-25 20:23:40 +08:00
刘祥超
fd6fa929de 修复连接数限制计算错误 2022-07-25 19:48:03 +08:00
刘祥超
73888c98a8 WAF多个相同Key的cc2统计规则不再重复累加 2022-07-25 09:34:34 +08:00
刘祥超
04ebfbea8a 节点状态中包含主程序位置 2022-07-21 15:07:12 +08:00
刘祥超
2b650fd285 API RPC配置增加disableUpdate,可以停用自动更新API节点 2022-07-21 14:06:38 +08:00
刘祥超
515a590681 地区封禁也可以使用自定义页面 2022-07-21 13:26:34 +08:00
刘祥超
62f9d1f09a 优化Firewalld添加端口方法,自动聚合连续的端口号 2022-07-21 11:53:23 +08:00
刘祥超
25c11f3d69 修改版本为v0.4.10 2022-07-20 18:14:12 +08:00
刘祥超
fde18c3b82 修复UserAgent中操作系统或浏览器版本中含有非UTF-8字符无法上传的问题 2022-07-20 16:12:46 +08:00
201 changed files with 3010 additions and 2606 deletions

View File

@@ -52,7 +52,6 @@ function build() {
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
# we support TOA on linux/amd64 only
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

View File

@@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"flag"
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/apps"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
@@ -19,7 +20,7 @@ import (
)
func main() {
app := apps.NewAppCmd().
var app = apps.NewAppCmd().
Version(teaconst.Version).
Product(teaconst.ProductName).
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog]").
@@ -67,24 +68,30 @@ func main() {
fmt.Println("done")
})
app.On("pprof", func() {
// TODO 自己指定端口
addr := "127.0.0.1:6060"
var flagSet = flag.NewFlagSet("pprof", flag.ExitOnError)
var addr string
flagSet.StringVar(&addr, "addr", "", "")
_ = flagSet.Parse(os.Args[2:])
if len(addr) == 0 {
addr = "127.0.0.1:6060"
}
logs.Println("starting with pprof '" + addr + "'...")
go func() {
err := http.ListenAndServe(addr, nil)
if err != nil {
logs.Println("[error]" + err.Error())
logs.Println("[ERROR]" + err.Error())
}
}()
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
app.On("dbstat", func() {
teaconst.EnableDBStat = true
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
app.On("trackers", func() {
@@ -154,7 +161,7 @@ func main() {
app.On("ip.drop", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS]")
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS] [--async]")
return
}
var ip = args[0]
@@ -168,6 +175,11 @@ func main() {
if ok {
timeoutSeconds = types.Int(timeout[0])
}
var async = false
_, ok = options["async"]
if ok {
async = true
}
fmt.Println("drop ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
var sock = gosock.NewTmpSock(teaconst.ProcessName)
@@ -176,6 +188,7 @@ func main() {
Params: map[string]interface{}{
"ip": ip,
"timeoutSeconds": timeoutSeconds,
"async": async,
},
})
if err != nil {
@@ -305,8 +318,23 @@ func main() {
}
}
})
app.On("bandwidth", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
var statsMap = maps.NewMap(reply.Params).Get("stats")
statsJSON, err := json.MarshalIndent(statsMap, "", " ")
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
fmt.Println(string(statsJSON))
})
app.Run(func() {
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
}

15
go.mod
View File

@@ -4,6 +4,7 @@ go 1.18
replace (
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
)
require (
@@ -16,10 +17,10 @@ require (
github.com/go-redis/redis/v8 v8.11.5
github.com/golang/protobuf v1.5.2
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5
github.com/klauspost/compress v1.15.8
github.com/mattn/go-sqlite3 v1.14.9
github.com/mdlayher/netlink v1.4.2
@@ -27,20 +28,22 @@ require (
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
github.com/shirou/gopsutil/v3 v3.22.2
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
golang.org/x/text v0.3.7
google.golang.org/grpc v1.45.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
rogchap.com/v8go v0.7.0
)
require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/webp v1.1.0 // indirect
github.com/chai2010/webp v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect

84
go.sum
View File

@@ -1,17 +1,13 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -20,8 +16,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -37,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -48,10 +43,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -83,20 +76,20 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55 h1:shQNx0flJFBwKsGE7Hs3bI2bDz+YF0zl/4qE8B2KRiY=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5 h1:SgKhrqRhWVpu0ZKxQM8MGjdhy7hlH3Xax8snwsZWSic=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -109,14 +102,10 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -150,23 +139,11 @@ github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb h1:2dC7L10LmTqlyMV
github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -180,9 +157,7 @@ github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtS
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
@@ -196,41 +171,30 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -259,21 +223,15 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -300,11 +258,10 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -316,7 +273,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
@@ -330,7 +286,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
@@ -339,7 +294,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
@@ -358,25 +312,19 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
rogchap.com/v8go v0.7.0 h1:kgjbiO4zE5itA962ze6Hqmbs4HgZbGzmueCXsZtremg=
rogchap.com/v8go v0.7.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs=

View File

@@ -89,7 +89,10 @@ func (this *LogWriter) Write(message string) {
}
}
this.c <- message
select {
case this.c <- message:
default:
}
}
func (this *LogWriter) Close() {

View File

@@ -96,13 +96,13 @@ func (this *FileList) Reset() error {
}
func (this *FileList) Add(hash string, item *Item) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
err := db.Add(hash, item)
err := db.AddAsync(hash, item)
if err != nil {
return err
}
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
}
func (this *FileList) Exist(hash string) (bool, error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// 如果Hash列表里不存在那么必然不存在
if !db.hashMap.Exist(hash) {
return false, nil
}
var item = this.memoryCache.Read(hash)
if item != nil {
return true, nil
@@ -225,9 +230,9 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
return err
}
if notFound {
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return err
return db.WrapError(err)
}
}
@@ -266,7 +271,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
_ = check
row := db.statStmt.QueryRow()
var row = db.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
@@ -291,13 +296,13 @@ func (this *FileList) Count() (int64, error) {
// IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
return db.IncreaseHit(hash)
return db.IncreaseHitAsync(hash)
}
// OnAdd 添加事件
@@ -326,17 +331,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
return fnv.HashString(hash) % CountFileDB
}
func (this *FileList) getDB(hash string) *FileListDB {
func (this *FileList) GetDB(hash string) *FileListDB {
return this.dbList[fnv.HashString(hash)%CountFileDB]
}
func (this *FileList) remove(hash string) (notFound bool, err error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// HashMap中不存在则确定不存在
if !db.hashMap.Exist(hash) {
return true, nil
}
defer db.hashMap.Delete(hash)
// 从缓存中删除
this.memoryCache.Delete(hash)
@@ -357,16 +368,16 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
return false, err
}
_, err = db.deleteByHashStmt.Exec(hash)
err = db.DeleteAsync(hash)
if err != nil {
return false, err
return false, db.WrapError(err)
}
atomic.AddInt64(&this.total, -1)
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return false, err
return false, db.WrapError(err)
}
if this.onRemove != nil {

View File

@@ -6,6 +6,8 @@ import (
"database/sql"
"errors"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/iwind/TeaGo/types"
@@ -16,9 +18,15 @@ import (
)
type FileListDB struct {
dbPath string
readDB *dbs.DB
writeDB *dbs.DB
writeBatch *dbs.Batch
hashMap *FileListHashMap
itemsTableName string
hitsTableName string
@@ -28,29 +36,48 @@ type FileListDB struct {
isReady bool
// cacheItems
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
selectByHashStmt *dbs.Stmt // 使用hash查询数据
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
insertSQL string
selectByHashStmt *dbs.Stmt // 使用hash查询数据
selectHashListStmt *dbs.Stmt
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
deleteByHashSQL string
statStmt *dbs.Stmt // 统计
purgeStmt *dbs.Stmt // 清理
deleteAllStmt *dbs.Stmt // 删除所有数据
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
// hits
insertHitStmt *dbs.Stmt // 写入数据
increaseHitStmt *dbs.Stmt // 增加点击量
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
insertHitSQL string // 写入数据
increaseHitSQL string // 增加点击量
deleteHitByHashSQL string // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
}
func NewFileListDB() *FileListDB {
return &FileListDB{}
return &FileListDB{
hashMap: NewFileListHashMap(),
}
}
func (this *FileListDB) Open(dbPath string) error {
this.dbPath = dbPath
// 动态调整Cache值
var cacheSize = 32000
var memoryGB = utils.SystemMemoryGB()
if memoryGB >= 8 {
cacheSize += 32000 * memoryGB / 8
}
// write db
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize)+"&_secure_delete=FAST")
if err != nil {
return errors.New("open write database failed: " + err.Error())
}
@@ -65,13 +92,22 @@ func (this *FileListDB) Open(dbPath string) error {
}**/
this.writeDB = dbs.NewDB(writeDB)
this.writeBatch = dbs.NewBatch(writeDB, 4)
this.writeBatch.OnFail(func(err error) {
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error())
})
goman.New(func() {
this.writeBatch.Exec()
})
if teaconst.EnableDBStat {
this.writeBatch.EnableStat(true)
this.writeDB.EnableStat(true)
}
// read db
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize))
if err != nil {
return errors.New("open read database failed: " + err.Error())
}
@@ -115,7 +151,8 @@ func (this *FileListDB) Init() error {
return err
}
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
if err != nil {
return err
}
@@ -125,7 +162,10 @@ func (this *FileListDB) Init() error {
return err
}
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
if err != nil {
return err
}
@@ -147,17 +187,11 @@ func (this *FileListDB) Init() error {
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
if err != nil {
return err
}
this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
@@ -166,6 +200,14 @@ func (this *FileListDB) Init() error {
this.isReady = true
// 加载HashMap
go func() {
err := this.hashMap.Load(this)
if err != nil {
remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
}
}()
return nil
}
@@ -177,17 +219,47 @@ func (this *FileListDB) Total() int64 {
return this.total
}
func (this *FileListDB) Add(hash string, item *Item) error {
func (this *FileListDB) AddAsync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
return nil
}
func (this *FileListDB) AddSync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
// 放入队列
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
if err != nil {
return this.WrapError(err)
}
return nil
}
func (this *FileListDB) DeleteAsync(hash string) error {
this.hashMap.Delete(hash)
this.writeBatch.Add(this.deleteByHashSQL, hash)
return nil
}
func (this *FileListDB) DeleteSync(hash string) error {
this.hashMap.Delete(hash)
_, err := this.deleteByHashStmt.Exec(hash)
if err != nil {
return err
}
return nil
}
@@ -246,10 +318,36 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
return
}
func (this *FileListDB) IncreaseHit(hash string) error {
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
rows, err := this.selectHashListStmt.Query(lastId)
if err != nil {
return nil, 0, err
}
var id int64
var hash string
for rows.Next() {
err = rows.Scan(&id, &hash)
if err != nil {
_ = rows.Close()
return
}
maxId = id
hashList = append(hashList, hash)
}
_ = rows.Close()
return
}
func (this *FileListDB) IncreaseHitAsync(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return err
this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
return nil
}
func (this *FileListDB) DeleteHitAsync(hash string) error {
this.writeBatch.Add(this.deleteHitByHashSQL, hash)
return nil
}
func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -262,7 +360,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
for {
result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
if err != nil {
return err
return this.WrapError(err)
}
affectedRows, err := result.RowsAffected()
if err != nil {
@@ -281,9 +379,11 @@ func (this *FileListDB) CleanAll() error {
_, err := this.deleteAllStmt.Exec()
if err != nil {
return err
return this.WrapError(err)
}
this.hashMap.Clean()
return nil
}
@@ -300,6 +400,9 @@ func (this *FileListDB) Close() error {
if this.selectByHashStmt != nil {
_ = this.selectByHashStmt.Close()
}
if this.selectHashListStmt != nil {
_ = this.selectHashListStmt.Close()
}
if this.deleteByHashStmt != nil {
_ = this.deleteByHashStmt.Close()
}
@@ -315,16 +418,6 @@ func (this *FileListDB) Close() error {
if this.listOlderItemsStmt != nil {
_ = this.listOlderItemsStmt.Close()
}
if this.insertHitStmt != nil {
_ = this.insertHitStmt.Close()
}
if this.increaseHitStmt != nil {
_ = this.increaseHitStmt.Close()
}
if this.deleteHitByHashStmt != nil {
_ = this.deleteHitByHashStmt.Close()
}
if this.lfuHitsStmt != nil {
_ = this.lfuHitsStmt.Close()
}
@@ -345,12 +438,23 @@ func (this *FileListDB) Close() error {
}
}
if this.writeBatch != nil {
this.writeBatch.Close()
}
if len(errStrings) == 0 {
return nil
}
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
}
func (this *FileListDB) WrapError(err error) error {
if err == nil {
return nil
}
return errors.New(err.Error() + "(file: " + this.dbPath + ")")
}
// 初始化
func (this *FileListDB) initTables(times int) error {
{
@@ -393,10 +497,10 @@ ON "` + this.itemsTableName + `" (
if dropErr == nil {
return this.initTables(times + 1)
}
return err
return this.WrapError(err)
}
return err
return this.WrapError(err)
}
}
@@ -421,10 +525,10 @@ ON "` + this.hitsTableName + `" (
if dropErr == nil {
return this.initTables(times + 1)
}
return err
return this.WrapError(err)
}
return err
return this.WrapError(err)
}
}

View File

@@ -0,0 +1,114 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"math/big"
"sync"
)
// FileListHashMap 文件Hash列表
type FileListHashMap struct {
m map[uint64]zero.Zero
locker sync.RWMutex
isAvailable bool
isReady bool
}
func NewFileListHashMap() *FileListHashMap {
return &FileListHashMap{
m: map[uint64]zero.Zero{},
isAvailable: false,
isReady: false,
}
}
func (this *FileListHashMap) Load(db *FileListDB) error {
// 如果系统内存过小,我们不缓存
if utils.SystemMemoryGB() < 3 {
return nil
}
this.isAvailable = true
var lastId int64
for {
hashList, maxId, err := db.ListHashes(lastId)
if err != nil {
return err
}
if len(hashList) == 0 {
break
}
this.AddHashes(hashList)
lastId = maxId
}
this.isReady = true
return nil
}
func (this *FileListHashMap) Add(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
this.m[this.bigInt(hash)] = zero.New()
this.locker.Unlock()
}
func (this *FileListHashMap) AddHashes(hashes []string) {
if !this.isAvailable {
return
}
this.locker.Lock()
for _, hash := range hashes {
this.m[this.bigInt(hash)] = zero.New()
}
this.locker.Unlock()
}
func (this *FileListHashMap) Delete(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
delete(this.m, this.bigInt(hash))
this.locker.Unlock()
}
func (this *FileListHashMap) Exist(hash string) bool {
if !this.isAvailable {
return true
}
if !this.isReady {
// 只有完全Ready时才能判断是否为false
return true
}
this.locker.RLock()
_, ok := this.m[this.bigInt(hash)]
this.locker.RUnlock()
return ok
}
func (this *FileListHashMap) Clean() {
this.locker.Lock()
this.m = map[uint64]zero.Zero{}
this.locker.Unlock()
}
func (this *FileListHashMap) IsReady() bool {
return this.isReady
}
func (this *FileListHashMap) bigInt(hash string) uint64 {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
return bigInt.Uint64()
}

View File

@@ -0,0 +1,91 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"math/big"
"runtime"
"strconv"
"testing"
)
func TestFileListHashMap_Memory(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = caches.NewFileListHashMap()
for i := 0; i < 1_000_000; i++ {
m.Add(stringutil.Md5(types.String(i)))
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_Memory2(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = map[uint64]zero.Zero{}
for i := 0; i < 1_000_000; i++ {
m[uint64(i)] = zero.New()
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_BigInt(t *testing.T) {
for _, s := range []string{"1", "2", "3", "123", "123456"} {
var hash = stringutil.Md5(s)
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
}
}
func TestFileListHashMap_Load(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
var m = caches.NewFileListHashMap()
err = m.Load(list.GetDB("abc"))
if err != nil {
t.Fatal(err)
}
m.Add("abc")
for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
t.Log(hash, "=>", m.Exist(hash))
}
}
func Benchmark_BigInt(b *testing.B) {
var hash = stringutil.Md5("123456")
b.ResetTimer()
for i := 0; i < b.N; i++ {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
_ = bigInt.Uint64()
}
}

View File

@@ -69,8 +69,8 @@ func TestFileList_Add_Many(t *testing.T) {
_ = list.Close()
}()
before := time.Now()
for i := 0; i < 100_000; i++ {
var before = time.Now()
for i := 0; i < 10_000_000; i++ {
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &caches.Item{
Key: u,
@@ -290,12 +290,12 @@ func TestFileList_Stat(t *testing.T) {
}
func TestFileList_Count(t *testing.T) {
list := caches.NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
var before = time.Now()
count, err := list.Count()
if err != nil {
t.Fatal(err)
@@ -367,6 +367,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
}

View File

@@ -6,7 +6,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"strconv"
"sync"
@@ -16,7 +15,7 @@ var SharedManager = NewManager()
func init() {
events.On(events.EventQuit, func() {
logs.Println("CACHE", "quiting cache manager")
remotelogs.Println("CACHE", "quiting cache manager")
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
})
}

View File

@@ -5,7 +5,7 @@ package caches
import (
"encoding/json"
"errors"
"io/ioutil"
"os"
)
// PartialRanges 内容分区范围定义
@@ -30,7 +30,7 @@ func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
}
func NewPartialRangesFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
@@ -116,12 +116,12 @@ func (this *PartialRanges) WriteToFile(path string) error {
if err != nil {
return errors.New("convert to json failed: " + err.Error())
}
return ioutil.WriteFile(path, data, 0666)
return os.WriteFile(path, data, 0666)
}
// ReadFromFile 从文件中读取
func (this *PartialRanges) ReadFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}

View File

@@ -65,11 +65,13 @@ var sharedWritingFileKeyLocker = sync.Mutex{}
var maxOpenFiles = NewMaxOpenFiles()
const maxOpenFilesSlowCost = 500 * time.Microsecond // 0.5ms
const maxOpenFilesSlowCost = 1000 * time.Microsecond // us
const protectingLoadWhenDump = false
// FileStorage 文件缓存
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
//
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
type FileStorage struct {
policy *serverconfigs.HTTPCachePolicy
options *serverconfigs.HTTPFileCacheStorage // 二级缓存

View File

@@ -8,7 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"io"
"net/http"
"runtime"
"strconv"
@@ -152,7 +152,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
"Last-Modified": []string{"Wed, 06 Jan 2021 10:03:29 GMT"},
"Server": []string{"CDN-Server"},
},
Body: ioutil.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
Body: io.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
}
for k, v := range resp.Header {

View File

@@ -95,7 +95,8 @@ func (this *MemoryStorage) Init() error {
// 启动定时Flush memory to disk任务
if this.parentStorage != nil {
// TODO 应该根据磁盘性能决定线程数
var threads = 1
// TODO 线程数应该可以在缓存策略和节点中设定
var threads = runtime.NumCPU()
for i := 0; i < threads; i++ {
goman.New(func() {
@@ -438,16 +439,18 @@ func (this *MemoryStorage) startFlush() {
if statCount == 100 {
statCount = 0
loadStat, err := load.Avg()
if err == nil && loadStat != nil {
if loadStat.Load1 > 10 {
writeDelayMS = 100
} else if loadStat.Load1 > 3 {
writeDelayMS = 50
} else if loadStat.Load1 > 2 {
writeDelayMS = 10
} else {
writeDelayMS = 0
if protectingLoadWhenDump {
loadStat, err := load.Avg()
if err == nil && loadStat != nil {
if loadStat.Load1 > 10 {
writeDelayMS = 100
} else if loadStat.Load1 > 3 {
writeDelayMS = 50
} else if loadStat.Load1 > 2 {
writeDelayMS = 10
} else {
writeDelayMS = 0
}
}
}
}

View File

@@ -8,6 +8,7 @@ import (
"runtime"
"runtime/debug"
"strconv"
"sync"
"testing"
"time"
)
@@ -304,3 +305,30 @@ func TestMemoryStorage_Stop(t *testing.T) {
t.Log(len(m))
}
func BenchmarkValuesMap(b *testing.B) {
var m = map[uint64]*MemoryItem{}
var count = 1_000_000
for i := 0; i < count; i++ {
m[uint64(i)] = &MemoryItem{
ExpiresAt: time.Now().Unix(),
}
}
b.Log(len(m))
var locker = sync.Mutex{}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
locker.Lock()
_, ok := m[uint64(rands.Int(0, 1_000_000))]
_ = ok
locker.Unlock()
locker.Lock()
delete(m, uint64(rands.Int(2, 1000000)))
locker.Unlock()
}
})
}

View File

@@ -5,7 +5,6 @@ package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"testing"
"time"
@@ -16,7 +15,7 @@ func TestPartialFileWriter_Write(t *testing.T) {
_ = os.Remove(path)
var reader = func() {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedBrotliReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedDeflateReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedGzipReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedZSTDReaderPool *ReaderPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -5,10 +5,46 @@ package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
stringutil "github.com/iwind/TeaGo/utils/string"
"strings"
"testing"
"time"
)
func TestBrotliWriter_LargeFile(t *testing.T) {
var data = []byte{}
for i := 0; i < 1024*1024; i++ {
data = append(data, stringutil.Rand(32)...)
}
t.Log(len(data)/1024/1024, "M")
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
t.Fatal(err)
}
var offset = 0
var size = 4096
for offset < len(data) {
_, err = writer.Write(data[offset : offset+size])
if err != nil {
t.Fatal(err)
}
offset += size
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
}
func BenchmarkBrotliWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/andybalholm/brotli"
"io"
@@ -11,6 +12,10 @@ import (
var sharedBrotliWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -4,6 +4,7 @@ package compressions
import (
"compress/flate"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -11,6 +12,10 @@ import (
var sharedDeflateWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -4,6 +4,7 @@ package compressions
import (
"compress/gzip"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -11,6 +12,10 @@ import (
var sharedGzipWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/klauspost/compress/zstd"
"io"
@@ -11,6 +12,10 @@ import (
var sharedZSTDWriterPool *WriterPool
func init() {
if teaconst.IsDaemon {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,20 +3,21 @@ package configs
import (
"github.com/iwind/TeaGo/Tea"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
)
// APIConfig 节点API配置
type APIConfig struct {
RPC struct {
Endpoints []string `yaml:"endpoints"`
Endpoints []string `yaml:"endpoints"`
DisableUpdate bool `yaml:"disableUpdate"`
} `yaml:"rpc"`
NodeId string `yaml:"nodeId"`
Secret string `yaml:"secret"`
}
func LoadAPIConfig() (*APIConfig, error) {
data, err := ioutil.ReadFile(Tea.ConfigFile("api.yaml"))
data, err := os.ReadFile(Tea.ConfigFile("api.yaml"))
if err != nil {
return nil, err
}
@@ -30,12 +31,12 @@ func LoadAPIConfig() (*APIConfig, error) {
return config, nil
}
// 保存到文件
// WriteFile 保存到文件
func (this *APIConfig) WriteFile(path string) error {
data, err := yaml.Marshal(this)
if err != nil {
return err
}
err = ioutil.WriteFile(path, data, 0666)
err = os.WriteFile(path, data, 0666)
return err
}

View File

@@ -1,11 +1,15 @@
package configs
package configs_test
import "testing"
import (
"github.com/TeaOSLab/EdgeNode/internal/configs"
_ "github.com/iwind/TeaGo/bootstrap"
"testing"
)
func TestLoadAPIConfig(t *testing.T) {
config, err := LoadAPIConfig()
config, err := configs.LoadAPIConfig()
if err != nil {
t.Fatal(err)
}
t.Log(config)
t.Logf("%+v", config)
}

View File

@@ -1,10 +1,11 @@
package configs
// 集群配置
// ClusterConfig 集群配置
type ClusterConfig struct {
RPC struct {
Endpoints []string `yaml:"endpoints"`
Endpoints []string `yaml:"endpoints"`
DisableUpdate bool `yaml:"disableUpdate"`
} `yaml:"rpc"`
ClusterId string `yaml:"clusterId"`
Secret string `yaml:"secret"`
Secret string `yaml:"secret"`
}

117
internal/conns/map.go Normal file
View File

@@ -0,0 +1,117 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package conns
import (
"net"
"sync"
)
var SharedMap = NewMap()
type Map struct {
m map[string]map[int]net.Conn // ip => { port => Conn }
locker sync.RWMutex
}
func NewMap() *Map {
return &Map{
m: map[string]map[int]net.Conn{},
}
}
func (this *Map) Add(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
this.m[ip] = map[int]net.Conn{
port: conn,
}
} else {
connMap[port] = conn
}
}
func (this *Map) Remove(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
return
}
delete(connMap, port)
if len(connMap) == 0 {
delete(this.m, ip)
}
}
func (this *Map) CountIPConns(ip string) int {
this.locker.RLock()
var l = len(this.m[ip])
this.locker.RUnlock()
return l
}
func (this *Map) CloseIPConns(ip string) {
var conns = []net.Conn{}
this.locker.RLock()
connMap, ok := this.m[ip]
// 复制防止在Close时产生并发冲突
if ok {
for _, conn := range connMap {
conns = append(conns, conn)
}
}
// 需要在Close之前结束防止死循环
this.locker.RUnlock()
if ok {
for _, conn := range conns {
_ = conn.Close()
}
// 这里不需要从 m 中删除,因为关闭时会自然触发回调
}
}
func (this *Map) AllConns() []net.Conn {
this.locker.RLock()
defer this.locker.RUnlock()
var result = []net.Conn{}
for _, m := range this.m {
for _, conn := range m {
result = append(result, conn)
}
}
return result
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.4.9"
Version = "0.5.2"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -2,7 +2,10 @@
package teaconst
import "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"os"
)
var (
// 流量统计
@@ -12,6 +15,7 @@ var (
NodeId int64 = 0
NodeIdString = ""
IsDaemon = len(os.Args) > 1 && os.Args[1] == "daemon"
GlobalProductName = nodeconfigs.DefaultProductName

View File

@@ -195,14 +195,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// new connections rate
var newConnectionsRate = tcpConfig.NewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = 100000
// new connections rate (minutely)
var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = 100000
}
}
var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
if newConnectionsMinutelyRateBlockTimeout < 0 {
newConnectionsMinutelyRateBlockTimeout = 0
}
// new connections rate (secondly)
var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = 10000
}
}
var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
if newConnectionsSecondlyRateBlockTimeout < 0 {
newConnectionsSecondlyRateBlockTimeout = 0
}
// 检查是否有变化
var hasChanges = false
@@ -215,7 +232,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
hasChanges = true
break
}
@@ -251,6 +272,7 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// TODO 让用户选择是drop还是reject
if maxConnectionsPerIP > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
var stderr = &bytes.Buffer{}
@@ -261,14 +283,47 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
if newConnectionsRate > 0 {
// TODO 思考是否有惩罚机制
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
// 超过一定速率就drop或者加入黑名单分钟
// TODO 让用户选择是drop还是reject
if newConnectionsMinutelyRate > 0 {
if newConnectionsMinutelyRateBlockTimeout > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
} else {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
// 超过一定速率就drop或者加入黑名单
// TODO 让用户选择是drop还是reject
if newConnectionsSecondlyRate > 0 {
if newConnectionsSecondlyRateBlockTimeout > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
} else {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
}
@@ -335,14 +390,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
for _, rule := range rules {
var pieces = this.decodeUserData(rule.UserData())
if len(pieces) != 4 {
if len(pieces) < 4 {
continue
}
if pieces[0] != "tcp" {
continue
}
switch pieces[2] {
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
err := chain.DeleteRule(rule)
if err != nil {
return err

View File

@@ -0,0 +1,47 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package firewalls
import (
"github.com/iwind/TeaGo/types"
"strings"
"sync"
"time"
)
type BaseFirewall struct {
locker sync.Mutex
latestIPTimes []string // [ip@time, ....]
}
// 检查是否在最近添加过
func (this *BaseFirewall) checkLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -3,6 +3,8 @@
package firewalls
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
@@ -10,15 +12,22 @@ import (
"strings"
)
type firewalldCmd struct {
cmd *exec.Cmd
denyIP string
}
type Firewalld struct {
BaseFirewall
isReady bool
exe string
cmdQueue chan *exec.Cmd
cmdQueue chan *firewalldCmd
}
func NewFirewalld() *Firewalld {
var firewalld = &Firewalld{
cmdQueue: make(chan *exec.Cmd, 4096),
cmdQueue: make(chan *firewalldCmd, 4096),
}
path, err := exec.LookPath("firewall-cmd")
@@ -40,13 +49,19 @@ func NewFirewalld() *Firewalld {
func (this *Firewalld) init() {
goman.New(func() {
for cmd := range this.cmdQueue {
for c := range this.cmdQueue {
var cmd = c.cmd
err := cmd.Run()
if err != nil {
if strings.HasPrefix(err.Error(), "Warning:") {
continue
}
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
} else {
// 关闭连接
if len(c.denyIP) > 0 {
conns.SharedMap.CloseIPConns(c.denyIP)
}
}
}
})
@@ -71,7 +86,25 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
return nil
}
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
return nil
}
func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol string) error {
for _, portRange := range portRanges {
var port = this.PortRangeString(portRange, protocol)
{
var cmd = exec.Command(this.exe, "--add-port="+port, "--permanent")
this.pushCmd(cmd, "")
}
{
var cmd = exec.Command(this.exe, "--add-port="+port)
this.pushCmd(cmd, "")
}
}
return nil
}
@@ -80,14 +113,44 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
return nil
}
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
return nil
}
func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol string) error {
var port = this.PortRangeString(portRange, protocol)
{
var cmd = exec.Command(this.exe, "--remove-port="+port, "--permanent")
this.pushCmd(cmd, "")
}
{
var cmd = exec.Command(this.exe, "--remove-port="+port)
this.pushCmd(cmd, "")
}
return nil
}
func (this *Firewalld) PortRangeString(portRange [2]int, protocol string) string {
if portRange[0] == portRange[1] {
return types.String(portRange[0]) + "/" + protocol
} else {
return types.String(portRange[0]) + "-" + types.String(portRange[1]) + "/" + protocol
}
}
func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -97,14 +160,20 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
this.pushCmd(cmd, ip)
return nil
}
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -114,7 +183,18 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
if async {
this.pushCmd(cmd, ip)
return nil
}
// 关闭连接
defer conns.SharedMap.CloseIPConns(ip)
err := cmd.Run()
if err != nil {
return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
}
return nil
}
@@ -122,6 +202,7 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -129,14 +210,14 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
for _, action := range []string{"reject", "drop"} {
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
this.pushCmd(cmd, "")
}
return nil
}
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
func (this *Firewalld) pushCmd(cmd *exec.Cmd, denyIP string) {
select {
case this.cmdQueue <- cmd:
case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
default:
// we discard the command
}

View File

@@ -23,7 +23,10 @@ type FirewallInterface interface {
RejectSourceIP(ip string, timeoutSeconds int) error
// DropSourceIP 丢弃某个源IP数据
DropSourceIP(ip string, timeoutSeconds int) error
// ip 要封禁的IP
// timeoutSeconds 过期时间
// async 是否异步
DropSourceIP(ip string, timeoutSeconds int, async bool) error
// RemoveSourceIP 删除某个源IP
RemoveSourceIP(ip string) error

View File

@@ -47,7 +47,7 @@ func (this *MockFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
}
// DropSourceIP 丢弃某个源IP数据
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
_ = ip
_ = timeoutSeconds
return nil

View File

@@ -7,8 +7,11 @@ package firewalls
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
"net"
@@ -21,9 +24,13 @@ import (
// check nft status, if being enabled we load it automatically
func init() {
if teaconst.IsDaemon {
return
}
if runtime.GOOS == "linux" {
var ticker = time.NewTicker(3 * time.Minute)
go func() {
goman.New(func() {
for range ticker.C {
// if already ready, we break
if nftablesIsReady {
@@ -48,7 +55,7 @@ func init() {
break
}
}
}()
})
}
}
@@ -74,9 +81,16 @@ func (this *nftablesTableDefinition) protocol() string {
return "ip"
}
type blockIPItem struct {
action string
ip string
timeoutSeconds int
}
func NewNFTablesFirewall() (*NFTablesFirewall, error) {
var firewall = &NFTablesFirewall{
conn: nftables.NewConn(),
conn: nftables.NewConn(),
dropIPQueue: make(chan *blockIPItem, 4096),
}
err := firewall.init()
if err != nil {
@@ -87,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
}
type NFTablesFirewall struct {
BaseFirewall
conn *nftables.Conn
isReady bool
version string
@@ -98,6 +114,8 @@ type NFTablesFirewall struct {
denyIPv6Set *nftables.Set
firewalld *Firewalld
dropIPQueue chan *blockIPItem
}
func (this *NFTablesFirewall) init() error {
@@ -243,6 +261,18 @@ func (this *NFTablesFirewall) init() error {
nftablesIsReady = true
nftablesInstance = this
goman.New(func() {
for ipItem := range this.dropIPQueue {
switch ipItem.action {
case "drop":
err = this.DropSourceIP(ipItem.ip, ipItem.timeoutSeconds, false)
if err != nil {
remotelogs.Warn("NFTABLES", "drop ip '"+ipItem.ip+"' failed: "+err.Error())
}
}
}
})
// load firewalld
var firewalld = NewFirewalld()
if firewalld.IsReady() {
@@ -307,16 +337,40 @@ func (this *NFTablesFirewall) AllowSourceIP(ip string) error {
// RejectSourceIP 拒绝某个源IP连接
// we did not create set for drop ip, so we reuse DropSourceIP() method here
func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
return this.DropSourceIP(ip, timeoutSeconds)
return this.DropSourceIP(ip, timeoutSeconds, true)
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
var data = net.ParseIP(ip)
if data == nil {
return errors.New("invalid ip '" + ip + "'")
}
// 尝试关闭连接
conns.SharedMap.CloseIPConns(ip)
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
if async {
select {
case this.dropIPQueue <- &blockIPItem{
action: "drop",
ip: ip,
timeoutSeconds: timeoutSeconds,
}:
default:
return errors.New("drop ip queue is full")
}
return nil
}
// 再次尝试关闭连接
defer conns.SharedMap.CloseIPConns(ip)
if strings.Contains(ip, ":") { // ipv6
if this.denyIPv6Set == nil {
return errors.New("ipv6 ip set is nil")
@@ -393,3 +447,35 @@ func (this *NFTablesFirewall) readVersion(nftPath string) string {
}
return versionMatches[1]
}
// 检查是否在最近添加过
func (this *NFTablesFirewall) existLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -51,7 +51,7 @@ func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) erro
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
return nil
}

View File

@@ -3,6 +3,7 @@
package goman
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"runtime"
"sync"
"time"
@@ -14,6 +15,10 @@ var instanceId = uint64(0)
// New 新创建goroutine
func New(f func()) {
if teaconst.IsDaemon {
return
}
_, file, line, _ := runtime.Caller(1)
go func() {
@@ -42,6 +47,10 @@ func New(f func()) {
// NewWithArgs 创建带有参数的goroutine
func NewWithArgs(f func(args ...interface{}), args ...interface{}) {
if teaconst.IsDaemon {
return
}
_, file, line, _ := runtime.Caller(1)
go func() {

View File

@@ -1,130 +0,0 @@
// 源码改自https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
package iplibrary
import (
"errors"
"io/ioutil"
"strconv"
"strings"
)
const (
IndexBlockLength = 12
)
var err error
type IP2Region struct {
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
lastIndexPtr int64
totalBlocks int64
dbData []byte
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
func getIpInfo(cityId int64, line []byte) *IpInfo {
lineSlice := strings.Split(string(line), "|")
ipInfo := &IpInfo{}
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
ipInfo.Country = lineSlice[0]
ipInfo.Region = lineSlice[1]
ipInfo.Province = lineSlice[2]
ipInfo.City = lineSlice[3]
ipInfo.ISP = lineSlice[4]
return ipInfo
}
func NewIP2Region(path string) (*IP2Region, error) {
var region = &IP2Region{}
region.dbData, err = ioutil.ReadFile(path)
if err != nil {
return nil, err
}
region.firstIndexPtr = region.ipLongAtOffset(0)
region.lastIndexPtr = region.ipLongAtOffset(4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
return region, nil
}
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
ip, err := ip2long(ipStr)
if err != nil {
return nil, err
}
h := this.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m*IndexBlockLength
sip := this.ipLongAtOffset(p)
if ip < sip {
h = m - 1
} else {
eip := this.ipLongAtOffset(p + 4)
if ip > eip {
l = m + 1
} else {
dataPtr = this.ipLongAtOffset(p + 8)
break
}
}
}
if dataPtr == 0 {
return nil, nil
}
dataLen := (dataPtr >> 24) & 0xFF
dataPtr = dataPtr & 0x00FFFFFF
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
}
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
return int64(this.dbData[offset]) |
int64(this.dbData[offset+1])<<8 |
int64(this.dbData[offset+2])<<16 |
int64(this.dbData[offset+3])<<24
}
func ip2long(IpStr string) (int64, error) {
bits := strings.Split(IpStr, ".")
if len(bits) != 4 {
return 0, errors.New("ip format error")
}
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24-8*i)
}
return sum, nil
}

View File

@@ -79,7 +79,7 @@ func TestIPItem_Memory(t *testing.T) {
for i := 0; i < 2_000_000; i ++ {
list.Add(&IPItem{
Type: "ip",
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long("192.168.1.1"),
IPTo: 0,
ExpiredAt: time.Now().Unix(),

View File

@@ -18,7 +18,8 @@ import (
type IPListDB struct {
db *sql.DB
itemTableName string
itemTableName string
deleteExpiredItemsStmt *sql.Stmt
deleteItemStmt *sql.Stmt
insertItemStmt *sql.Stmt
@@ -28,6 +29,8 @@ type IPListDB struct {
cleanTicker *time.Ticker
dir string
isClosed bool
}
func NewIPListDB() (*IPListDB, error) {
@@ -51,11 +54,21 @@ func (this *IPListDB) init() error {
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = this.dir + "/ip_list.db"
_ = os.Remove(path + "-shm")
_ = os.Remove(path + "-wal")
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
db.SetMaxOpenConns(1)
//_, err = db.Exec("VACUUM")
//if err != nil {
// return err
//}
this.db = db
// 初始化数据库
@@ -117,6 +130,7 @@ ON "` + this.itemTableName + `" (
goman.New(func() {
events.On(events.EventQuit, func() {
_ = this.Close()
this.cleanTicker.Stop()
})
@@ -133,11 +147,19 @@ ON "` + this.itemTableName + `" (
// DeleteExpiredItems 删除过期的条目
func (this *IPListDB) DeleteExpiredItems() error {
if this.isClosed {
return nil
}
_, err := this.deleteExpiredItemsStmt.Exec(time.Now().Unix() - 7*86400)
return err
}
func (this *IPListDB) AddItem(item *pb.IPItem) error {
if this.isClosed {
return nil
}
_, err := this.deleteItemStmt.Exec(item.Id)
if err != nil {
return err
@@ -147,6 +169,10 @@ func (this *IPListDB) AddItem(item *pb.IPItem) error {
}
func (this *IPListDB) ReadItems(offset int64, size int64) (items []*pb.IPItem, err error) {
if this.isClosed {
return
}
rows, err := this.selectItemsStmt.Query(offset, size)
if err != nil {
return nil, err
@@ -169,12 +195,16 @@ func (this *IPListDB) ReadItems(offset int64, size int64) (items []*pb.IPItem, e
// ReadMaxVersion 读取当前最大版本号
func (this *IPListDB) ReadMaxVersion() int64 {
row := this.selectMaxVersionStmt.QueryRow()
if this.isClosed {
return 0
}
var row = this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err = row.Scan(&version)
err := row.Scan(&version)
if err != nil {
return 0
}
@@ -182,6 +212,8 @@ func (this *IPListDB) ReadMaxVersion() int64 {
}
func (this *IPListDB) Close() error {
this.isClosed = true
if this.db != nil {
_ = this.deleteExpiredItemsStmt.Close()
_ = this.deleteItemStmt.Close()

View File

@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.AddItem(&pb.IPItem{
Id: 1,
IpFrom: "192.168.1.101",
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.Close()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
@@ -53,6 +60,11 @@ func TestIPListDB_ReadItems(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer func() {
_ = db.Close()
}()
items, err := db.ReadItems(0, 2)
if err != nil {
t.Fatal(err)

View File

@@ -144,7 +144,7 @@ func TestIPList_Contains(t *testing.T) {
list := NewIPList()
for i := 0; i < 255; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0,
@@ -152,7 +152,7 @@ func TestIPList_Contains(t *testing.T) {
}
for i := 0; i < 255; i++ {
list.AddDelay(&IPItem{
Id: int64(1000 + i),
Id: uint64(1000 + i),
IPFrom: utils.IP2Long("192.167.2." + strconv.Itoa(i)),
})
}
@@ -172,7 +172,7 @@ func TestIPList_Contains_Many(t *testing.T) {
list := NewIPList()
for i := 0; i < 1_000_000; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
ExpiredAt: 0,
@@ -217,7 +217,7 @@ func TestIPList_ContainsIPStrings(t *testing.T) {
list := NewIPList()
for i := 0; i < 255; i++ {
list.Add(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0,
@@ -305,7 +305,7 @@ func BenchmarkIPList_Contains(b *testing.B) {
var list = NewIPList()
for i := 1; i < 200_000; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60,

View File

@@ -1,13 +0,0 @@
package iplibrary
type LibraryInterface interface {
// Load 加载数据库文件
Load(dbPath string) error
// Lookup 查询IP
// 返回结果有可能为空
Lookup(ip string) (*Result, error)
// Close 关闭数据库文件
Close()
}

View File

@@ -1,83 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"net"
"strings"
)
type IP2RegionLibrary struct {
db *IP2Region
}
func (this *IP2RegionLibrary) Load(dbPath string) error {
db, err := NewIP2Region(dbPath)
if err != nil {
return err
}
this.db = db
return nil
}
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
// 暂不支持IPv6
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
if this.db == nil {
return nil, errors.New("library has not been loaded")
}
defer func() {
// 防止panic发生
err := recover()
if err != nil {
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
}
}()
info, err := this.db.MemorySearch(ip)
if err != nil {
return nil, err
}
if info == nil {
return nil, nil
}
if info.Country == "0" {
info.Country = ""
}
if info.Region == "0" {
info.Region = ""
}
if info.Province == "0" {
info.Province = ""
}
if info.City == "0" {
info.City = ""
}
if info.ISP == "0" {
info.ISP = ""
}
return &Result{
CityId: info.CityId,
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
ISP: info.ISP,
}, nil
}
func (this *IP2RegionLibrary) Close() {
}

View File

@@ -1,113 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
"sync"
"testing"
"time"
)
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
var mem = &runtime.MemStats{}
runtime.ReadMemStats(mem)
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var mem2 = &runtime.MemStats{}
runtime.ReadMemStats(mem2)
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
}
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"8.8.9.9"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var count = 4000
var wg = sync.WaitGroup{}
wg.Add(count)
for i := 0; i < count; i++ {
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
}()
}
wg.Done()
t.Log("ok")
}
func TestIP2RegionLibrary_Memory(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 1_000_000; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
var library = &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_, _ = library.Lookup("8.8.8.8")
}
}

View File

@@ -1,95 +0,0 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/files"
"github.com/iwind/TeaGo/types"
"regexp"
"strings"
)
var SharedManager = NewManager()
var SharedLibrary LibraryInterface
func init() {
events.On(events.EventLoaded, func() {
// 初始化
library, err := SharedManager.Load()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
return
}
SharedLibrary = library
})
}
type Manager struct {
code string
}
func NewManager() *Manager {
return &Manager{}
}
func (this *Manager) Load() (LibraryInterface, error) {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return nil, err
}
config := nodeConfig.GlobalConfig
if config == nil {
config = &serverconfigs.GlobalConfig{}
}
// 当前正在使用的IP库代号
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
dir := Tea.Root + "/resources/ipdata/" + code
var lastVersion int64 = -1
lastFilename := ""
for _, file := range files.NewFile(dir).List() {
filename := file.Name()
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
if reg.MatchString(filename) { // 先查找有版本号的
result := reg.FindStringSubmatch(filename)
version := types.Int64(result[1])
if version > lastVersion {
lastVersion = version
lastFilename = filename
}
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
if lastVersion == -1 {
lastFilename = filename
lastVersion = 0
}
}
}
if len(lastFilename) == 0 {
return nil, errors.New("ip library file not found")
}
var libraryPtr LibraryInterface
switch code {
case serverconfigs.IPLibraryTypeIP2Region:
libraryPtr = &IP2RegionLibrary{}
default:
return nil, errors.New("invalid ip library code '" + code + "'")
}
err = libraryPtr.Load(dir + "/" + lastFilename)
if err != nil {
return nil, err
}
return libraryPtr, nil
}

View File

@@ -1,156 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCityManager = NewCityManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCityManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCityManager.Stop()
})
}
// CityManager 中国省份信息管理
type CityManager struct {
ticker *time.Ticker
cacheFile string
cityMap map[string]int64 // provinceName_cityName => cityName
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCityManager() *CityManager {
return &CityManager{
cacheFile: Tea.Root + "/configs/region_city.json.cache",
cityMap: map[string]int64{},
}
}
func (this *CityManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("City_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
}
}
func (this *CityManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
this.locker.RLock()
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *CityManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.cityMap = m
}
return nil
}
// 更新城市信息
func (this *CityManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCityRPC().FindAllEnabledRegionCities(rpcClient.Context(), &pb.FindAllEnabledRegionCitiesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, city := range resp.RegionCities {
for _, code := range city.Codes {
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.cityMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,14 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewCityManager(t *testing.T) {
var manager = NewCityManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup(16, "许昌市"))
}

View File

@@ -1,154 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCountryManager = NewCountryManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCountryManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCountryManager.Stop()
})
}
// CountryManager 国家/地区信息管理
type CountryManager struct {
ticker *time.Ticker
cacheFile string
countryMap map[string]int64 // countryName => countryId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCountryManager() *CountryManager {
return &CountryManager{
cacheFile: Tea.Root + "/configs/region_country.json.cache",
countryMap: map[string]int64{},
}
}
func (this *CountryManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
}
}
func (this *CountryManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
this.locker.RLock()
countryId, _ = this.countryMap[countryName]
this.locker.RUnlock()
return countryId
}
// 从缓存中读取
func (this *CountryManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.countryMap = m
}
return nil
}
// 更新国家信息
func (this *CountryManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCountryRPC().FindAllEnabledRegionCountries(rpcClient.Context(), &pb.FindAllEnabledRegionCountriesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, country := range resp.RegionCountries {
for _, code := range country.Codes {
m[code] = country.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.countryMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestCountryManager_load(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop(t *testing.T) {
manager := NewCountryManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop_skip(t *testing.T) {
manager := NewCountryManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestCountryManager_Lookup(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
}
func BenchmarkCountryManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewCountryManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("中国")
}
}

View File

@@ -43,7 +43,7 @@ type IPListManager struct {
func NewIPListManager() *IPListManager {
return &IPListManager{
pageSize: 500,
pageSize: 1000,
listMap: map[int64]*IPList{},
}
}
@@ -111,15 +111,19 @@ func (this *IPListManager) init() {
var size int64 = 1000
for {
items, err := db.ReadItems(offset, size)
var l = len(items)
if err != nil {
remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
} else {
if len(items) == 0 {
if l == 0 {
break
}
this.processItems(items, false)
if int64(l) < size {
break
}
}
offset += int64(len(items))
offset += int64(l)
}
}
}
@@ -144,7 +148,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
if err != nil {
return false, err
}
itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
Version: this.version,
Size: this.pageSize,
})
@@ -155,7 +159,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
}
return false, err
}
items := itemsResp.IpItems
var items = itemsResp.IpItems
if len(items) == 0 {
return false, nil
}
@@ -183,7 +187,6 @@ func (this *IPListManager) FindList(listId int64) *IPList {
}
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
this.locker.Lock()
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
var list *IPList
@@ -203,11 +206,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
list = GlobalWhiteIPList
}
} else { // 其他List
this.locker.Lock()
list = this.listMap[item.ListId]
this.locker.Unlock()
}
if list == nil {
list = NewIPList()
this.locker.Lock()
this.listMap[item.ListId] = list
this.locker.Unlock()
}
changedLists[list] = zero.New()
@@ -246,8 +253,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
changedList.Sort()
}
this.locker.Unlock()
if fromRemote {
var latestVersion = items[len(items)-1].Version
if latestVersion > this.version {

View File

@@ -1,155 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedProviderManager = NewProviderManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProviderManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProviderManager.Stop()
})
}
// ProviderManager 中国省份信息管理
type ProviderManager struct {
ticker *time.Ticker
cacheFile string
providerMap map[string]int64 // name => id
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProviderManager() *ProviderManager {
return &ProviderManager{
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
providerMap: map[string]int64{},
}
}
func (this *ProviderManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
}
}
func (this *ProviderManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
this.locker.RLock()
providerId, _ = this.providerMap[providerName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *ProviderManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.providerMap = m
}
return nil
}
// 更新服务商信息
func (this *ProviderManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProviderRPC().FindAllEnabledRegionProviders(rpcClient.Context(), &pb.FindAllEnabledRegionProvidersRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, provider := range resp.RegionProviders {
for _, code := range provider.Codes {
m[code] = provider.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.providerMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,15 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewProviderManager(t *testing.T) {
var manager = NewProviderManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("阿里云"))
t.Log(manager.Lookup("阿里云2"))
}

View File

@@ -1,161 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
const (
ChinaCountryId int64 = 1
)
var SharedProvinceManager = NewProvinceManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProvinceManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProvinceManager.Stop()
})
}
// ProvinceManager 中国省份信息管理
type ProvinceManager struct {
ticker *time.Ticker
cacheFile string
provinceMap map[string]int64 // provinceName => provinceId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProvinceManager() *ProvinceManager {
return &ProvinceManager{
cacheFile: Tea.Root + "/configs/region_province.json.cache",
provinceMap: map[string]int64{},
}
}
func (this *ProvinceManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
}
}
func (this *ProvinceManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
this.locker.RLock()
provinceId, _ = this.provinceMap[provinceName]
this.locker.RUnlock()
return provinceId
}
// 从缓存中读取
func (this *ProvinceManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.provinceMap = m
}
return nil
}
// 更新省份信息
func (this *ProvinceManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProvinceRPC().FindAllEnabledRegionProvincesWithCountryId(rpcClient.Context(), &pb.FindAllEnabledRegionProvincesWithCountryIdRequest{
RegionCountryId: ChinaCountryId,
})
if err != nil {
return err
}
m := map[string]int64{}
for _, province := range resp.RegionProvinces {
for _, code := range province.Codes {
m[code] = province.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.provinceMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestProvinceManager_load(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop(t *testing.T) {
manager := NewProvinceManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop_skip(t *testing.T) {
manager := NewProvinceManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestProvinceManager_Lookup(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
}
func BenchmarkProvinceManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewProvinceManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("安徽省")
}
}

View File

@@ -1,26 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestManager_Load(t *testing.T) {
dbs.NotifyReady()
manager := NewManager()
lib, err := manager.Load()
if err != nil {
t.Fatal(err)
}
t.Log(lib.Lookup("1.2.3.4"))
t.Log(lib.Lookup("2.3.4.5"))
t.Log(lib.Lookup("200.200.200.200"))
t.Log(lib.Lookup("202.106.0.20"))
}
func TestNewManager(t *testing.T) {
dbs.NotifyReady()
t.Log(SharedLibrary)
}

View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"os"
"time"
)
var SharedUpdater = NewUpdater()
func init() {
events.On(events.EventStart, func() {
goman.New(func() {
SharedUpdater.Start()
})
})
events.On(events.EventQuit, func() {
SharedUpdater.Stop()
})
}
// Updater IP库更新程序
type Updater struct {
ticker *time.Ticker
}
// NewUpdater 获取新对象
func NewUpdater() *Updater {
return &Updater{}
}
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新因为通常不需要更新IP库
this.ticker = time.NewTicker(1 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
}
}
func (this *Updater) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
// 单次任务
func (this *Updater) loop() error {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return err
}
if nodeConfig.GlobalConfig == nil {
return nil
}
code := nodeConfig.GlobalConfig.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
if err != nil {
return err
}
lib := libraryResp.IpLibrary
if lib == nil || lib.File == nil {
return nil
}
typeInfo := serverconfigs.FindIPLibraryWithType(code)
if typeInfo == nil {
return errors.New("invalid ip library code '" + code + "'")
}
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
// 是否已经存在
_, err = os.Stat(path)
if err == nil {
return nil
}
// 开始下载
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
if err != nil {
return err
}
chunkIds := fileChunkIdsResp.FileChunkIds
if len(chunkIds) == 0 {
return nil
}
isOk := false
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer func() {
// 如果保存不成功就直接删除
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
for _, chunkId := range chunkIds {
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
chunk := chunkResp.FileChunk
if chunk == nil {
continue
}
_, err = fp.Write(chunk.Data)
if err != nil {
return err
}
}
err = fp.Close()
if err != nil {
return err
}
// 重新加载
library, err := SharedManager.Load()
if err != nil {
return err
}
SharedLibrary = library
isOk = true
return nil
}

View File

@@ -1,18 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestUpdater_loop(t *testing.T) {
dbs.NotifyReady()
updater := NewUpdater()
err := updater.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -4,6 +4,7 @@ package metrics
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"strconv"
"sync"
@@ -11,7 +12,15 @@ import (
var SharedManager = NewManager()
func init() {
events.On(events.EventQuit, func() {
SharedManager.Quit()
})
}
type Manager struct {
isQuiting bool
tasks map[int64]*Task // itemId => *Task
categoryTasks map[string][]*Task // category => []*Task
locker sync.RWMutex
@@ -29,6 +38,10 @@ func NewManager() *Manager {
}
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
if this.isQuiting {
return
}
this.locker.Lock()
defer this.locker.Unlock()
@@ -101,6 +114,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
// Add 添加数据
func (this *Manager) Add(obj MetricInterface) {
if this.isQuiting {
return
}
this.locker.RLock()
for _, task := range this.categoryTasks[obj.MetricCategory()] {
task.Add(obj)
@@ -119,3 +136,17 @@ func (this *Manager) HasTCPMetrics() bool {
func (this *Manager) HasUDPMetrics() bool {
return this.hasUDPMetrics
}
// Quit 退出管理器
func (this *Manager) Quit() {
this.isQuiting = true
remotelogs.Println("METRIC_MANAGER", "quit")
this.locker.Lock()
for _, task := range this.tasks {
_ = task.Stop()
}
this.tasks = map[int64]*Task{}
this.locker.Unlock()
}

View File

@@ -16,6 +16,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
@@ -23,18 +24,19 @@ import (
"time"
)
const MaxQueueSize = 2048
const MaxQueueSize = 256 // TODO 可以配置,可以在单个任务里配置
// Task 单个指标任务
// 数据库存储:
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 上传每个服务中排序最前面的 N 个数据
//
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 只上传每个服务中排序最前面的 N 个数据
type Task struct {
item *serverconfigs.MetricItemConfig
isLoaded bool
@@ -88,7 +90,11 @@ func (this *Task) Init() error {
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
_ = os.Remove(path + "-shm")
_ = os.Remove(path + "-wal")
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
@@ -372,7 +378,9 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
for _, serverId := range serverIds {
for _, currentTime := range times {
idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
var t = trackers.Begin("[METRIC]SELECT_TOP_STMT")
rows, err := this.selectTopStmt.Query(serverId, this.item.Version, currentTime)
t.End()
if err != nil {
return nil, err
}
@@ -421,7 +429,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
return nil, err
}
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
MetricStats: pbStats,
Time: currentTime,
ServerId: serverId,

View File

@@ -69,7 +69,7 @@ func (this *ValueQueue) Loop() error {
}
for value := range this.valuesChan {
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
Item: value.Item,
ValueJSON: value.ValueJSON,
CreatedAt: value.CreatedAt,

View File

@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
if err != nil {
statusErr, ok:= status.FromError(err)
statusErr, ok := status.FromError(err)
if ok {
logs.Println(statusErr.Code())
}

View File

@@ -73,7 +73,7 @@ func (this *APIStream) loop() error {
cancelFunc()
}()
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
nodeStream, err := rpcClient.NodeRPC.NodeStream(ctx)
if err != nil {
if this.isQuiting {
return nil

View File

@@ -5,12 +5,14 @@ package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
"net"
"os"
@@ -22,6 +24,8 @@ import (
// ClientConn 客户端连接
type ClientConn struct {
BaseClientConn
once sync.Once
isTLS bool
@@ -31,29 +35,28 @@ type ClientConn struct {
isLO bool // 是否为环路
hasResetSYNFlood bool
BaseClientConn
}
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
if quickClose {
// TCP
tcpConn, ok := conn.(*net.TCPConn)
if ok {
// TODO 可以在配置中设置此值
_ = tcpConn.SetLinger(nodeconfigs.DefaultTCPLinger)
}
}
func NewClientConn(rawConn net.Conn, isTLS bool, quickClose bool) net.Conn {
// 是否为环路
var remoteAddr = conn.RemoteAddr().String()
var remoteAddr = rawConn.RemoteAddr().String()
var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
return &ClientConn{
BaseClientConn: BaseClientConn{rawConn: conn},
var conn = &ClientConn{
BaseClientConn: BaseClientConn{rawConn: rawConn},
isTLS: isTLS,
isLO: isLO,
}
if quickClose {
// TODO 可以在配置中设置此值
_ = conn.SetLinger(nodeconfigs.DefaultTCPLinger)
}
// 加入到Map
conns.SharedMap.Add(conn)
return conn
}
func (this *ClientConn) Read(b []byte) (n int, err error) {
@@ -110,11 +113,10 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
func (this *ClientConn) Write(b []byte) (n int, err error) {
n, err = this.rawConn.Write(b)
if n > 0 {
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
// 统计当前服务带宽
if this.serverId > 0 {
if !this.isLO { // 环路不统计带宽,避免缓存预热等行为产生带宽
if !this.isLO || Tea.IsTesting() { // 环路不统计带宽,避免缓存预热等行为产生带宽
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
}
}
@@ -129,9 +131,11 @@ func (this *ClientConn) Close() error {
err := this.rawConn.Close()
// 单个服务并发数限制
if this.hasLimit {
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
}
// 不能加条件限制,因为服务配置随时有变化
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
// 从conn map中移除
conns.SharedMap.Remove(this)
return err
}
@@ -178,6 +182,11 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
if timeout <= 0 {
timeout = 600
}
// 关闭当前连接
_ = this.SetLinger(0)
_ = this.Close()
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击当前1分钟"+types.String(result)+"次空连接")
}
}

View File

@@ -2,7 +2,10 @@
package nodes
import "net"
import (
"crypto/tls"
"net"
)
type BaseClientConn struct {
rawConn net.Conn
@@ -42,6 +45,17 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
// SetServerId 设置服务ID
func (this *BaseClientConn) SetServerId(serverId int64) {
this.serverId = serverId
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
nativeConn, ok := conn.NetConn().(ClientConnInterface)
if ok {
nativeConn.SetServerId(serverId)
}
case *ClientConn:
conn.SetServerId(serverId)
}
}
// ServerId 读取当前连接绑定的服务ID
@@ -52,6 +66,17 @@ func (this *BaseClientConn) ServerId() int64 {
// SetUserId 设置所属服务的用户ID
func (this *BaseClientConn) SetUserId(userId int64) {
this.userId = userId
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
nativeConn, ok := conn.NetConn().(ClientConnInterface)
if ok {
nativeConn.SetUserId(userId)
}
case *ClientConn:
conn.SetUserId(userId)
}
}
// UserId 获取当前连接所属服务的用户ID
@@ -66,9 +91,20 @@ func (this *BaseClientConn) RawIP() string {
}
// TCPConn 转换为TCPConn
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
conn, ok := this.rawConn.(*net.TCPConn)
return conn, ok
func (this *BaseClientConn) TCPConn() (tcpConn *net.TCPConn, ok bool) {
// 设置包装前连接
switch conn := this.rawConn.(type) {
case *tls.Conn:
var internalConn = conn.NetConn()
clientConn, ok := internalConn.(*ClientConn)
if ok {
return clientConn.TCPConn()
}
tcpConn, ok = internalConn.(*net.TCPConn)
default:
tcpConn, ok = this.rawConn.(*net.TCPConn)
}
return
}
// SetLinger 设置Linger

View File

@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"net"
"time"
)
// ClientListener 客户端网络监听
@@ -42,10 +43,28 @@ func (this *ClientListener) Accept() (net.Conn, error) {
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err == nil {
canGoNext, _ := iplibrary.AllowIP(ip, 0)
var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
expiresAt, ok := waf.SharedIPBlackList.ContainsExpires(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if ok {
var timeout = expiresAt - time.Now().Unix()
if timeout > 0 {
canGoNext = false
if !canGoNext || beingDenied {
if timeout > 3600 {
timeout = 3600
}
// 使用本地防火墙延长封禁
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
// 这里 int(int64) 转换的前提是限制了 timeout <= 3600否则将有整型溢出的风险
_ = fw.DropSourceIP(ip, int(timeout), true)
}
}
}
}
if !canGoNext {
tcpConn, ok := conn.(*net.TCPConn)
if ok {
_ = tcpConn.SetLinger(0)
@@ -53,14 +72,6 @@ func (this *ClientListener) Accept() (net.Conn, error) {
_ = conn.Close()
// 使用本地防火墙延长封禁
if beingDenied {
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
_ = fw.DropSourceIP(ip, 60)
}
}
return this.Accept()
}
}

View File

@@ -96,7 +96,7 @@ Loop:
this.rpcClient = client
}
_, err := this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
_, err := this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
if err != nil {
// 是否包含了invalid UTF-8
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
@@ -105,7 +105,7 @@ Loop:
}
// 重新提交
_, err = this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
return err
}

View File

@@ -55,7 +55,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
// logs.PrintAsJSON(accessLog)
//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
accessLog,
}})
if err != nil {
@@ -99,7 +99,7 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
accessLog,
}})
if err != nil {

View File

@@ -15,7 +15,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"io"
"io/ioutil"
"net"
"net/http"
"regexp"
@@ -82,7 +81,7 @@ func (this *HTTPCacheTaskManager) Start() {
if rpcClient != nil {
for taskReq := range this.taskQueue {
_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
_, err := rpcClient.ServerRPC.PurgeServerCache(rpcClient.Context(), taskReq)
if err != nil {
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
}
@@ -105,8 +104,12 @@ func (this *HTTPCacheTaskManager) Loop() error {
return err
}
resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
resp, err := rpcClient.HTTPCacheTaskKeyRPC.FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
if err != nil {
// 忽略连接错误
if rpc.IsConnError(err) {
return nil
}
return err
}
@@ -132,7 +135,7 @@ func (this *HTTPCacheTaskManager) Loop() error {
pbResults = append(pbResults, pbResult)
}
_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
_, err = rpcClient.HTTPCacheTaskKeyRPC.UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
if err != nil {
return err
}
@@ -236,7 +239,7 @@ func (this *HTTPCacheTaskManager) fetchKey(key *pb.HTTPCacheTaskKey) error {
}()
// 读取内容,以便于生成缓存
_, _ = io.Copy(ioutil.Discard, resp.Body)
_, _ = io.Copy(io.Discard, resp.Body)
// 处理502
if resp.StatusCode == http.StatusBadGateway {

View File

@@ -54,6 +54,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
}
var key = origin.UniqueKey() + "@" + originAddr
var isLnRequest = origin.Id == 0
this.locker.RLock()
client, found := this.clientsMap[key]
@@ -101,6 +102,17 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
idleConns = numberCPU * 8
}
// 可以判断为Ln节点请求
if isLnRequest {
maxConnections *= 8
idleConns *= 8
idleTimeout *= 4
} else if sharedNodeConfig != nil && sharedNodeConfig.Level > 1 {
// Ln节点可以适当增加连接数
maxConnections *= 2
idleConns *= 2
}
// TLS通讯
var tlsConfig = &tls.Config{
InsecureSkipVerify: true,
@@ -149,6 +161,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 5 * time.Second,
TLSClientConfig: tlsConfig,
ReadBufferSize: 8 * 1024,
Proxy: nil,
},
}

View File

@@ -38,7 +38,7 @@ func TestHTTPClientPool_Client(t *testing.T) {
}
func TestHTTPClientPool_cleanClients(t *testing.T) {
origin := &serverconfigs.OriginConfig{
var origin = &serverconfigs.OriginConfig{
Id: 1,
Version: 2,
Addr: &serverconfigs.NetworkAddressConfig{Host: "127.0.0.1", PortRange: "1234"},
@@ -48,8 +48,7 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
t.Fatal(err)
}
pool := NewHTTPClientPool()
pool.clientExpiredDuration = 2 * time.Second
var pool = NewHTTPClientPool()
for i := 0; i < 10; i++ {
t.Log("get", i)

View File

@@ -6,10 +6,10 @@ import (
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -17,7 +17,6 @@ import (
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
@@ -151,7 +150,7 @@ func (this *HTTPRequest) Do() {
// Web配置
err := this.configureWeb(this.ReqServer.Web, true, 0)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to configure the server", "配置服务失败", false)
this.doEnd()
return
}
@@ -284,12 +283,12 @@ func (this *HTTPRequest) doBegin() {
this.web.AccessLogRef.IsOn &&
this.web.AccessLogRef.ContainsField(serverconfigs.HTTPAccessLogFieldRequestBody) {
var err error
this.requestBodyData, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
this.requestBodyData, err = io.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
this.write50x(err, http.StatusBadGateway, "Failed to read request body for access log", "为访问日志读取请求Body失败", false)
return
}
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
this.RawReq.Body = io.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
}
// 跳转
@@ -678,7 +677,7 @@ func (this *HTTPRequest) Format(source string) string {
case "remoteAddrValue":
return this.requestRemoteAddr(false)
case "rawRemoteAddr":
addr := this.RawReq.RemoteAddr
var addr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
@@ -929,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
// geo
if prefix == "geo" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "country.name":
if result != nil {
return result.Country
if result != nil && result.IsOk() {
return result.CountryName()
}
return ""
case "country.id":
if result != nil {
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
if result != nil && result.IsOk() {
return types.String(result.CountryId())
}
return "0"
case "province.name":
if result != nil {
return result.Province
if result != nil && result.IsOk() {
return result.ProvinceName()
}
return ""
case "province.id":
if result != nil {
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
if result != nil && result.IsOk() {
return types.String(result.ProvinceId())
}
return "0"
case "city.name":
if result != nil {
return result.City
if result != nil && result.IsOk() {
return result.CityName()
}
return ""
case "city.id":
if result != nil {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 {
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
} else {
return "0"
}
if result != nil && result.IsOk() {
return types.String(result.CityId())
}
return "0"
case "town.name":
if result != nil && result.IsOk() {
return result.TownName()
}
return ""
case "town.id":
if result != nil && result.IsOk() {
return types.String(result.TownId())
}
return "0"
}
@@ -972,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
// ips
if prefix == "isp" {
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
var result = iplib.LookupIP(this.requestRemoteAddr(true))
switch suffix {
case "name":
if result != nil {
return result.ISP
if result != nil && result.IsOk() {
return result.ProviderName()
}
case "id":
if result != nil {
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
if result != nil && result.IsOk() {
return types.String(result.ProviderId())
}
return "0"
}
@@ -1099,7 +1103,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
}
// Remote-Addr
remoteAddr := this.RawReq.RemoteAddr
var remoteAddr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
if supportVar {
@@ -1168,7 +1172,7 @@ func (this *HTTPRequest) requestRemoteUser() string {
// Path 请求的URL中路径部分
func (this *HTTPRequest) Path() string {
uri, err := url.ParseRequestURI(this.rawURI)
uri, err := url.ParseRequestURI(this.uri)
if err != nil {
return ""
}
@@ -1316,7 +1320,7 @@ func (this *HTTPRequest) RemoteAddr() string {
}
func (this *HTTPRequest) RawRemoteAddr() string {
addr := this.RawReq.RemoteAddr
var addr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(addr)
if err == nil {
addr = host
@@ -1390,12 +1394,12 @@ func (this *HTTPRequest) Cookie(name string) string {
return c.Value
}
// DeleteHeader 删除Header
// DeleteHeader 删除请求Header
func (this *HTTPRequest) DeleteHeader(name string) {
this.RawReq.Header.Del(name)
}
// SetHeader 设置Header
// SetHeader 设置请求Header
func (this *HTTPRequest) SetHeader(name string, values []string) {
this.RawReq.Header[name] = values
}
@@ -1424,11 +1428,16 @@ func (this *HTTPRequest) Done() {
func (this *HTTPRequest) Close() {
this.Done()
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
if requestConn == nil {
return
}
lingerConn, ok := requestConn.(LingerConn)
if ok {
_ = lingerConn.SetLinger(0)
}
conn, ok := requestConn.(net.Conn)
if ok {
_ = conn.Close()
@@ -1712,12 +1721,17 @@ func (this *HTTPRequest) canIgnore(err error) bool {
}
// 客户端主动取消
if err == errWritingToClient || err == context.Canceled || err == io.ErrShortWrite || strings.Contains(err.Error(), "write: connection timed out") || strings.Contains(err.Error(), "write: broken pipe") {
if err == errWritingToClient ||
err == context.Canceled ||
err == io.ErrShortWrite ||
strings.Contains(err.Error(), "write: connection") ||
strings.Contains(err.Error(), "write: broken pipe") ||
strings.Contains(err.Error(), "write tcp") {
return true
}
// HTTP/2流错误
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
if err.Error() == "http2: stream closed" || strings.Contains(err.Error(), "stream error") || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
return true
}

View File

@@ -21,7 +21,7 @@ func (this *HTTPRequest) doACME() (shouldStop bool) {
return false
}
keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
keyResp, err := rpcClient.ACMEAuthenticationRPC.FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
if err != nil {
remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
return false

View File

@@ -5,7 +5,7 @@ package nodes
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"io/ioutil"
"io"
"net/http"
)
@@ -19,41 +19,52 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
continue
}
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
if !ref.AuthPolicy.MatchRequest(this.RawReq) {
continue
}
ok, newURI, uriChanged, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
subReq.TLS = this.RawReq.TLS
subReq.RemoteAddr = this.RawReq.RemoteAddr
subReq.Host = this.RawReq.Host
subReq.Proto = this.RawReq.Proto
subReq.ProtoMinor = this.RawReq.ProtoMinor
subReq.ProtoMajor = this.RawReq.ProtoMajor
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
subReq.Body = io.NopCloser(bytes.NewReader([]byte{}))
subReq.Header.Set("Referer", this.URL())
var writer = NewEmptyResponseWriter(this.writer)
this.doSubRequest(writer, subReq)
return writer.StatusCode(), nil
}, this.Format)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
return
}
if b {
if ok {
if uriChanged {
this.uri = newURI
}
this.tags = append(this.tags, ref.AuthPolicy.Type)
return
} else {
// Basic Auth比较特殊
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.ReqHost
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
if ok {
var headerValue = "Basic realm=\""
if len(method.Realm) > 0 {
headerValue += method.Realm
} else {
headerValue += this.ReqHost
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
headerValue += "\""
if len(method.Charset) > 0 {
headerValue += ", charset=\"" + method.Charset + "\""
}
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
}
this.writer.WriteHeader(http.StatusUnauthorized)
this.tags = append(this.tags, ref.AuthPolicy.Type)
return true
}
}

View File

@@ -44,12 +44,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
// 检查服务独立的缓存条件
refType := ""
for _, cacheRef := range this.web.Cache.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
if !cacheRef.IsOn {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
if cacheRef.IsReverse {
return
}
@@ -61,12 +60,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
// 检查策略默认的缓存条件
for _, cacheRef := range cachePolicy.CacheRefs {
if !cacheRef.IsOn ||
cacheRef.Conds == nil ||
!cacheRef.Conds.HasRequestConds() {
if !cacheRef.IsOn {
continue
}
if cacheRef.Conds.MatchRequest(this.Format) {
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
if cacheRef.IsReverse {
return
}

View File

@@ -1,32 +1,59 @@
package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"net/http"
"strings"
)
const httpStatusPageTemplate = `<!DOCTYPE html>
<html>
<head>
<title>${status} ${statusMessage}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body>
<h1>${status} ${statusMessage}</h1>
<p>${message}</p>
<address>Request ID: ${requestId}.</address>
</body>
</html>`
func (this *HTTPRequest) write404() {
if this.doPage(http.StatusNotFound) {
this.writeCode(http.StatusNotFound)
}
func (this *HTTPRequest) writeCode(statusCode int) {
if this.doPage(statusCode) {
return
}
this.processResponseHeaders(http.StatusNotFound)
this.writer.WriteHeader(http.StatusNotFound)
_, _ = this.writer.Write([]byte("404 page not found: '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
var pageContent = configutils.ParseVariables(httpStatusPageTemplate, func(varName string) (value string) {
switch varName {
case "status":
return types.String(statusCode)
case "statusMessage":
return http.StatusText(statusCode)
case "requestId":
return this.requestId
case "message":
return "" // 空
}
return "${" + varName + "}"
})
this.processResponseHeaders(statusCode)
this.writer.WriteHeader(statusCode)
_, _ = this.writer.Write([]byte(pageContent))
}
func (this *HTTPRequest) writeCode(code int) {
if this.doPage(code) {
return
}
this.processResponseHeaders(code)
this.writer.WriteHeader(code)
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
}
func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
func (this *HTTPRequest) write50x(err error, statusCode int, enMessage string, zhMessage string, canTryStale bool) {
if err != nil {
this.addError(err)
}
@@ -37,7 +64,7 @@ func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
this.web.Cache.Stale != nil &&
this.web.Cache.Stale.IsOn &&
(len(this.web.Cache.Stale.Status) == 0 || lists.ContainsInt(this.web.Cache.Stale.Status, statusCode)) {
ok := this.doCacheRead(true)
var ok = this.doCacheRead(true)
if ok {
return
}
@@ -47,7 +74,34 @@ func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
if this.doPage(statusCode) {
return
}
// 内置HTML模板
var pageContent = configutils.ParseVariables(httpStatusPageTemplate, func(varName string) (value string) {
switch varName {
case "status":
return types.String(statusCode)
case "statusMessage":
return http.StatusText(statusCode)
case "requestId":
return this.requestId
case "message":
var acceptLanguages = this.RawReq.Header.Get("Accept-Language")
if len(acceptLanguages) > 0 {
var index = strings.Index(acceptLanguages, ",")
if index > 0 {
var firstLanguage = acceptLanguages[:index]
if firstLanguage == "zh-CN" {
return "网站出了一点小问题,原因:" + zhMessage + "。"
}
}
}
return "The site is unavailable now, cause: " + enMessage + "."
}
return "${" + varName + "}"
})
this.processResponseHeaders(statusCode)
this.writer.WriteHeader(statusCode)
_, _ = this.writer.Write([]byte(types.String(statusCode) + " " + http.StatusText(statusCode) + " (Request Id: " + this.requestId + ")"))
_, _ = this.writer.Write([]byte(pageContent))
}

View File

@@ -81,7 +81,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
client, err := fcgi.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to create Fastcgi pool", "Fastcgi池生成失败", false)
return
}
@@ -159,13 +159,13 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
resp, stderr, err := client.Call(fcgiReq)
if err != nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to read Fastcgi", "读取Fastcgi失败", false)
return
}
if len(stderr) > 0 {
err := errors.New("Fastcgi Error: " + strings.TrimSpace(string(stderr)) + " script: " + maps.NewMap(params).GetString("SCRIPT_FILENAME"))
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to read Fastcgi", "读取Fastcgi失败", false)
return
}

View File

@@ -16,6 +16,6 @@ func (this *HTTPRequest) checkLnRequest() bool {
return false
}
func (this *HTTPRequest) getLnOrigin() *serverconfigs.OriginConfig {
return nil
func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
return nil, 0, false
}

View File

@@ -34,17 +34,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
}
return this.RawReq.ContentLength + hl, true
case "${countConnection}":
metricNewConnMapLocker.Lock()
_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
if ok {
delete(metricNewConnMap, this.RawReq.RemoteAddr)
}
metricNewConnMapLocker.Unlock()
if ok {
return 1, true
} else {
return 0, false
}
return 1, true
}
return 0, false
}

View File

@@ -21,10 +21,32 @@ func (this *HTTPRequest) doReverseProxy() {
return
}
var retries = 3
var failedOriginIds []int64
var failedLnNodeIds []int64
for i := 0; i < retries; i++ {
originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1)
if !shouldRetry {
break
}
if originId > 0 {
failedOriginIds = append(failedOriginIds, originId)
}
if lnNodeId > 0 {
failedLnNodeIds = append(failedLnNodeIds, lnNodeId)
}
}
}
// 请求源站
func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool) (originId int64, lnNodeId int64, shouldRetry bool) {
// 对URL的处理
var stripPrefix = this.reverseProxy.StripPrefix
var requestURI = this.reverseProxy.RequestURI
var requestURIHasVariables = this.reverseProxy.RequestURIHasVariables()
var oldURI = this.uri
var requestHost = ""
if this.reverseProxy.RequestHostType == serverconfigs.RequestHostTypeCustomized {
@@ -41,24 +63,39 @@ func (this *HTTPRequest) doReverseProxy() {
var origin *serverconfigs.OriginConfig
// 二级节点
var hasMultipleLnNodes = false
if this.cacheRef != nil {
origin = this.getLnOrigin()
origin, lnNodeId, hasMultipleLnNodes = this.getLnOrigin(failedLnNodeIds)
if origin != nil {
// 强制变更原来访问的域名
requestHost = this.ReqHost
}
// 回源Header中去除If-None-Match和If-Modified-Since
if !this.cacheRef.EnableIfNoneMatch {
this.DeleteHeader("If-None-Match")
}
if !this.cacheRef.EnableIfModifiedSince {
this.DeleteHeader("If-Modified-Since")
}
}
// 自定义源站
if origin == nil {
origin = this.reverseProxy.NextOrigin(requestCall)
if !isFirstTry {
origin = this.reverseProxy.AnyOrigin(requestCall, failedOriginIds)
}
if origin == nil {
origin = this.reverseProxy.NextOrigin(requestCall)
}
requestCall.CallResponseCallbacks(this.writer)
if origin == nil {
err := errors.New(this.URL() + ": no available origin sites for reverse proxy")
remotelogs.ServerError(this.ReqServer.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "No origin site yet", "尚未配置源站", true)
return
}
originId = origin.Id
if len(origin.StripPrefix) > 0 {
stripPrefix = origin.StripPrefix
@@ -80,7 +117,7 @@ func (this *HTTPRequest) doReverseProxy() {
if origin.Addr == nil {
err := errors.New(this.URL() + ": Origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Origin site did not has a valid address", "源站尚未配置地址", true)
return
}
this.RawReq.URL.Scheme = origin.Addr.Protocol.Primary().Scheme()
@@ -132,7 +169,7 @@ func (this *HTTPRequest) doReverseProxy() {
if originHostIndex < 0 {
var originErr = errors.New(this.URL() + ": Invalid origin address '" + originAddr + "', lacking port")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", originErr.Error())
this.write50x(originErr, http.StatusBadGateway, true)
this.write50x(originErr, http.StatusBadGateway, "No port in origin site address", "源站地址中没有配置端口", true)
return
}
originAddr = originAddr[:originHostIndex+1] + types.String(this.requestServerPort())
@@ -211,16 +248,10 @@ func (this *HTTPRequest) doReverseProxy() {
client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol, this.reverseProxy.FollowRedirects)
if err != nil {
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Create client failed: "+err.Error())
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to create origin site client", "构造源站客户端失败", true)
return
}
// 在HTTP/2下需要防止因为requestBody而导致Content-Length为空的问题
if this.RawReq.ProtoMajor == 2 && this.RawReq.ContentLength == 0 {
_ = this.RawReq.Body.Close()
this.RawReq.Body = nil
}
// 开始请求
resp, err := client.Do(this.RawReq)
if err != nil {
@@ -230,18 +261,35 @@ func (this *HTTPRequest) doReverseProxy() {
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+": Request origin server failed: "+err.Error())
} else if httpErr.Err != context.Canceled {
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
// 是否需要重试
if (originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) && !isLastRetry {
shouldRetry = true
this.uri = oldURI // 恢复备份
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
if httpErr.Err != io.EOF {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
}
return
}
if httpErr.Timeout() {
this.write50x(err, http.StatusGatewayTimeout, true)
this.write50x(err, http.StatusGatewayTimeout, "Read origin site timeout", "源站读取超时", true)
} else if httpErr.Temporary() {
this.write50x(err, http.StatusServiceUnavailable, true)
this.write50x(err, http.StatusServiceUnavailable, "Origin site unavailable now", "源站当前不可用", true)
} else {
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
}
if httpErr.Err != io.EOF {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
@@ -264,7 +312,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
if !isClientError {
this.write50x(err, http.StatusBadGateway, true)
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
}
}
if resp != nil && resp.Body != nil {
@@ -410,4 +458,6 @@ func (this *HTTPRequest) doReverseProxy() {
if (err == nil || err == io.EOF) && (closeErr == nil || closeErr == io.EOF) {
this.writer.SetOk()
}
return
}

View File

@@ -114,7 +114,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
return
} else {
this.write50x(err, http.StatusInternalServerError, true)
this.write50x(err, http.StatusInternalServerError, "Failed to stat the file", "查看文件统计信息失败", true)
if !this.canIgnore(err) {
logs.Error(err)
}
@@ -145,7 +145,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
return
} else {
this.write50x(err, http.StatusInternalServerError, true)
this.write50x(err, http.StatusInternalServerError, "Failed to stat the file", "查看文件统计信息失败", true)
if !this.canIgnore(err) {
logs.Error(err)
}
@@ -285,7 +285,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
fileReader, err := os.OpenFile(filePath, os.O_RDONLY, 0444)
if err != nil {
this.write50x(err, http.StatusInternalServerError, true)
this.write50x(err, http.StatusInternalServerError, "Failed to open the file", "试图打开文件失败", true)
return true
}

View File

@@ -35,7 +35,7 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
resp, err := client.Do(req)
if err != nil {
remotelogs.Error("HTTP_REQUEST_URL", req.URL.String()+": "+err.Error())
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to read url", "读取URL失败", false)
return
}
defer func() {

View File

@@ -180,14 +180,14 @@ var httpRequestTimestamp int64
var httpRequestId int32 = 1_000_000
func httpRequestNextId() string {
var unixTime = utils.UnixTimeMilli()
unixTime, unixTimeString := utils.UnixTimeMilliString()
if unixTime > httpRequestTimestamp {
atomic.StoreInt32(&httpRequestId, 1_000_000)
httpRequestTimestamp = unixTime
}
// timestamp + requestId + nodeId
return strconv.FormatInt(unixTime, 10) + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
// timestamp + nodeId + requestId
return unixTimeString + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
}
// 检查是否可以接受某个编码

View File

@@ -2,6 +2,7 @@ package nodes
import (
"bytes"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -11,7 +12,6 @@ import (
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"io"
"io/ioutil"
"net/http"
)
@@ -162,54 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// 检查地区封禁
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
if iplibrary.SharedLibrary != nil {
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
if err != nil {
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
} else if result != nil {
// 检查国家级别封禁
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
regionConfig := firewallPolicy.Inbound.Region
if regionConfig.IsNotEmpty() {
for _, remoteAddr := range remoteAddrs {
var result = iplib.LookupIP(remoteAddr)
if result != nil && result.IsOk() {
// 检查国家/地区级别封禁
var countryId = result.CountryId()
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
this.firewallPolicyId = firewallPolicy.Id
this.writer.WriteHeader(http.StatusForbidden)
this.writer.Close()
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
// 检查省份封禁
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
return true, false
}
this.writer.WriteHeader(http.StatusForbidden)
this.writer.Close()
// 检查省份封禁
var provinceId = result.ProvinceId()
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
this.firewallPolicyId = firewallPolicy.Id
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
this.writeCode(http.StatusForbidden)
this.writer.Flush()
this.writer.Close()
return true, false
}
// 停止日志
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
return true, false
}
}
}
@@ -357,7 +351,7 @@ func (this *HTTPRequest) WAFSetCacheBody(body []byte) {
// WAFReadBody 读取Body
func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
if this.RawReq.ContentLength > 0 {
data, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, max))
data, err = io.ReadAll(io.LimitReader(this.RawReq.Body, max))
}
return
@@ -366,7 +360,7 @@ func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
// WAFRestoreBody 恢复Body
func (this *HTTPRequest) WAFRestoreBody(data []byte) {
if len(data) > 0 {
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(data), this.RawReq.Body))
this.RawReq.Body = io.NopCloser(io.MultiReader(bytes.NewBuffer(data), this.RawReq.Body))
}
}
@@ -378,6 +372,8 @@ func (this *HTTPRequest) WAFServerId() int64 {
// WAFClose 关闭连接
func (this *HTTPRequest) WAFClose() {
this.Close()
// 这里不要强关IP所有连接避免因为单个服务而影响所有
}
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {

View File

@@ -43,7 +43,7 @@ func (this *HTTPRequest) doWebsocket(requestHost string) {
// TODO 增加N次错误重试重试的时候需要尝试不同的源站
originConn, _, err := OriginConnect(this.origin, this.requestServerPort(), this.RawReq.RemoteAddr, requestHost)
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
this.write50x(err, http.StatusBadGateway, "Failed to connect origin site", "源站连接失败", false)
// 增加失败次数
SharedOriginStateManager.Fail(this.origin, requestHost, this.reverseProxy, func() {
@@ -65,13 +65,13 @@ func (this *HTTPRequest) doWebsocket(requestHost string) {
err = this.RawReq.Write(originConn)
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
this.write50x(err, http.StatusBadGateway, "Failed to write request to origin site", "源站请求初始化失败", false)
return
}
clientConn, _, err := this.writer.Hijack()
if err != nil || clientConn == nil {
this.write50x(err, http.StatusInternalServerError, false)
this.write50x(err, http.StatusInternalServerError, "Failed to get origin site connection", "获取源站连接失败", false)
return
}
defer func() {

View File

@@ -26,7 +26,6 @@ import (
_ "image/jpeg"
_ "image/png"
"io"
"io/ioutil"
"net"
"net/http"
"net/textproto"
@@ -524,7 +523,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
this.webpOriginContentType = contentType
this.webpIsEncoding = true
resp.Body = ioutil.NopCloser(&bytes.Buffer{})
resp.Body = io.NopCloser(&bytes.Buffer{})
this.delayRead = true
this.Header().Del("Content-Length")
@@ -1008,30 +1007,35 @@ func (this *HTTPWriter) finishWebP() {
Exact: true,
})
} else if gifImage != nil {
anim := gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
anim.WebPAnimEncoderOptions.SetKmin(9)
anim.WebPAnimEncoderOptions.SetKmax(17)
defer anim.ReleaseMemory()
webpConfig := gowebp.NewWebpConfig()
var webpConfig = gowebp.NewWebpConfig()
//webpConfig.SetLossless(1)
webpConfig.SetQuality(f)
var timeline = 0
var lastErr error
for i, img := range gifImage.Image {
err = anim.AddFrame(img, timeline, webpConfig)
if err != nil {
break
// 有错误直接跳过
lastErr = err
err = nil
}
timeline += gifImage.Delay[i] * 10
}
if err == nil {
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
if lastErr != nil {
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
}
err = anim.AddFrame(nil, timeline, webpConfig)
if err == nil {
err = anim.Encode(this.writer)
}
anim.ReleaseMemory()
}
if err != nil && !this.req.canIgnore(err) {

View File

@@ -0,0 +1,103 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
"io"
"os"
)
type IPLibraryUpdater struct {
}
func NewIPLibraryUpdater() *IPLibraryUpdater {
return &IPLibraryUpdater{}
}
// DataDir 文件目录
func (this *IPLibraryUpdater) DataDir() string {
// data/
var dir = Tea.Root + "/data"
stat, err := os.Stat(dir)
if err == nil && stat.IsDir() {
return dir
}
err = os.Mkdir(dir, 0666)
if err == nil {
return dir
}
remotelogs.Error("IP_LIBRARY_UPDATER", "create directory '"+dir+"' failed: "+err.Error())
// 如果不能创建 data/ 目录,那么使用临时目录
return os.TempDir()
}
// FindLatestFile 检查最新的IP库文件
func (this *IPLibraryUpdater) FindLatestFile() (code string, fileId int64, err error) {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return "", 0, err
}
resp, err := rpcClient.IPLibraryArtifactRPC.FindPublicIPLibraryArtifact(rpcClient.Context(), &pb.FindPublicIPLibraryArtifactRequest{})
if err != nil {
return "", 0, err
}
var artifact = resp.IpLibraryArtifact
if artifact == nil {
return
}
return artifact.Code, artifact.FileId, nil
}
// DownloadFile 下载文件
func (this *IPLibraryUpdater) DownloadFile(fileId int64, writer io.Writer) error {
if fileId <= 0 {
return errors.New("invalid fileId: " + types.String(fileId))
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
chunkIdsResp, err := rpcClient.FileChunkRPC.FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: fileId})
if err != nil {
return err
}
for _, chunkId := range chunkIdsResp.FileChunkIds {
chunkResp, err := rpcClient.FileChunkRPC.DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
var chunk = chunkResp.FileChunk
if chunk == nil {
return errors.New("can not find file chunk with chunk id '" + types.String(chunkId) + "'")
}
_, err = writer.Write(chunk.Data)
if err != nil {
return err
}
}
return nil
}
// LogInfo 普通日志
func (this *IPLibraryUpdater) LogInfo(message string) {
remotelogs.Println("IP_LIBRARY_UPDATER", message)
}
// LogError 错误日志
func (this *IPLibraryUpdater) LogError(err error) {
if err == nil {
return
}
remotelogs.Error("IP_LIBRARY_UPDATER", err.Error())
}

View File

@@ -5,7 +5,6 @@ import (
"crypto/tls"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"golang.org/x/net/http2"
"io"
@@ -13,14 +12,11 @@ import (
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
var httpErrorLogger = log.New(io.Discard, "", 0)
var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool
var metricNewConnMapLocker = &sync.Mutex{}
type contextKey struct {
key string
@@ -55,23 +51,10 @@ func (this *HTTPListener) Serve() error {
switch state {
case http.StateNew:
atomic.AddInt64(&this.countActiveConnections, 1)
// 为指标存储连接信息
if sharedNodeConfig.HasHTTPConnectionMetrics() {
metricNewConnMapLocker.Lock()
metricNewConnMap[conn.RemoteAddr().String()] = zero.New()
metricNewConnMapLocker.Unlock()
}
case http.StateActive, http.StateIdle, http.StateHijacked:
// Nothing to do
case http.StateClosed:
atomic.AddInt64(&this.countActiveConnections, -1)
// 移除指标存储连接信息
// 因为中途配置可能有改变,所以暂时不添加条件
metricNewConnMapLocker.Lock()
delete(metricNewConnMap, conn.RemoteAddr().String())
metricNewConnMapLocker.Unlock()
}
},
ConnContext: func(ctx context.Context, conn net.Conn) context.Context {

View File

@@ -5,11 +5,14 @@ import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"net/url"
"os/exec"
"regexp"
@@ -31,15 +34,19 @@ type ListenerManager struct {
retryListenerMap map[string]*Listener // 需要重试的监听器 addr => Listener
ticker *time.Ticker
lastPortStrings string
firewalld *firewalls.Firewalld
lastPortStrings string
lastTCPPortRanges [][2]int
lastUDPPortRanges [][2]int
}
// NewListenerManager 获取新对象
func NewListenerManager() *ListenerManager {
manager := &ListenerManager{
var manager = &ListenerManager{
listenersMap: map[string]*Listener{},
retryListenerMap: map[string]*Listener{},
ticker: time.NewTicker(1 * time.Minute),
firewalld: firewalls.NewFirewalld(),
}
// 提升测试效率
@@ -147,7 +154,7 @@ func (this *ListenerManager) Start(node *nodeconfigs.NodeConfig) error {
}
// 加入到firewalld
this.addToFirewalld(groupAddrs)
go this.addToFirewalld(groupAddrs)
return nil
}
@@ -226,8 +233,14 @@ func (this *ListenerManager) addToFirewalld(groupAddrs []string) {
return
}
if this.firewalld == nil || !this.firewalld.IsReady() {
return
}
// 组合端口号
var ports = []string{}
var portStrings = []string{}
var udpPorts = []int{}
var tcpPorts = []int{}
for _, addr := range groupAddrs {
var protocol = "tcp"
if strings.HasPrefix(addr, "udp") {
@@ -237,52 +250,72 @@ func (this *ListenerManager) addToFirewalld(groupAddrs []string) {
var lastIndex = strings.LastIndex(addr, ":")
if lastIndex > 0 {
var portString = addr[lastIndex+1:]
ports = append(ports, portString+"/"+protocol)
portStrings = append(portStrings, portString+"/"+protocol)
switch protocol {
case "tcp":
tcpPorts = append(tcpPorts, types.Int(portString))
case "udp":
udpPorts = append(udpPorts, types.Int(portString))
}
}
}
if len(ports) == 0 {
if len(portStrings) == 0 {
return
}
// 检查是否有变化
sort.Strings(ports)
var newPortStrings = strings.Join(ports, ",")
sort.Strings(portStrings)
var newPortStrings = strings.Join(portStrings, ",")
if newPortStrings == this.lastPortStrings {
return
}
this.lastPortStrings = newPortStrings
firewallCmd, err := exec.LookPath("firewall-cmd")
if err != nil || len(firewallCmd) == 0 {
return
remotelogs.Println("FIREWALLD", "opening ports automatically ...")
defer func() {
remotelogs.Println("FIREWALLD", "open ports successfully")
}()
// 合并端口
var tcpPortRanges = utils.MergePorts(tcpPorts)
var udpPortRanges = utils.MergePorts(udpPorts)
defer func() {
this.lastTCPPortRanges = tcpPortRanges
this.lastUDPPortRanges = udpPortRanges
}()
// 删除老的不存在的端口
var tcpPortRangesMap = map[string]bool{}
var udpPortRangesMap = map[string]bool{}
for _, portRange := range tcpPortRanges {
tcpPortRangesMap[this.firewalld.PortRangeString(portRange, "tcp")] = true
}
for _, portRange := range udpPortRanges {
udpPortRangesMap[this.firewalld.PortRangeString(portRange, "udp")] = true
}
// 检查状态
err = exec.Command(firewallCmd, "--state").Run()
if err != nil {
return
}
remotelogs.Println("FIREWALLD", "open ports automatically")
for _, port := range ports {
{
// TODO 需要支持sudo
var cmd = exec.Command(firewallCmd, "--add-port="+port, "--permanent")
err = cmd.Run()
if err != nil {
remotelogs.Warn("FIREWALLD", "'"+cmd.String()+"': "+err.Error())
return
}
}
{
// TODO 需要支持sudo
var cmd = exec.Command(firewallCmd, "--add-port="+port)
err = cmd.Run()
if err != nil {
remotelogs.Warn("FIREWALLD", "'"+cmd.String()+"': "+err.Error())
return
}
for _, portRange := range this.lastTCPPortRanges {
var s = this.firewalld.PortRangeString(portRange, "tcp")
_, ok := tcpPortRangesMap[s]
if ok {
continue
}
remotelogs.Println("FIREWALLD", "remove port '"+s+"'")
_ = this.firewalld.RemovePortRangePermanently(portRange, "tcp")
}
for _, portRange := range this.lastUDPPortRanges {
var s = this.firewalld.PortRangeString(portRange, "udp")
_, ok := udpPortRangesMap[s]
if ok {
continue
}
remotelogs.Println("FIREWALLD", "remove port '"+s+"'")
_ = this.firewalld.RemovePortRangePermanently(portRange, "udp")
}
// 添加新的
_ = this.firewalld.AllowPortRangesPermanently(tcpPortRanges, "tcp")
_ = this.firewalld.AllowPortRangesPermanently(udpPortRanges, "udp")
}

View File

@@ -4,6 +4,7 @@ import (
"crypto/tls"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/stats"
@@ -108,8 +109,9 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
// 记录域名排行
tlsConn, ok := conn.(*tls.Conn)
var recordStat = false
var serverName = ""
if ok {
var serverName = tlsConn.ConnectionState().ServerName
serverName = tlsConn.ConnectionState().ServerName
if len(serverName) > 0 {
// 统计
stats.SharedTrafficStatManager.Add(server.Id, serverName, 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
@@ -122,8 +124,9 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
stats.SharedTrafficStatManager.Add(server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
}
originConn, err := this.connectOrigin(server.Id, server.ReverseProxy, conn.RemoteAddr().String())
originConn, err := this.connectOrigin(server.Id, serverName, server.ReverseProxy, conn.RemoteAddr().String())
if err != nil {
_ = conn.Close()
return err
}
@@ -218,37 +221,63 @@ func (this *TCPListener) Close() error {
}
// 连接源站
func (this *TCPListener) connectOrigin(serverId int64, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
func (this *TCPListener) connectOrigin(serverId int64, requestHost string, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
if reverseProxy == nil {
return nil, errors.New("no reverse proxy config")
}
var requestCall = shared.NewRequestCall()
requestCall.Domain = requestHost
var retries = 3
var addr string
var failedOriginIds []int64
for i := 0; i < retries; i++ {
var origin = reverseProxy.NextOrigin(nil)
var origin *serverconfigs.OriginConfig
if len(failedOriginIds) > 0 {
origin = reverseProxy.AnyOrigin(requestCall, failedOriginIds)
}
if origin == nil {
origin = reverseProxy.NextOrigin(requestCall)
}
if origin == nil {
continue
}
// 回源主机名
var requestHost = ""
if len(reverseProxy.RequestHost) > 0 {
requestHost = reverseProxy.RequestHost
}
if len(origin.RequestHost) > 0 {
requestHost = origin.RequestHost
} else if len(reverseProxy.RequestHost) > 0 {
requestHost = reverseProxy.RequestHost
}
conn, addr, err = OriginConnect(origin, this.port, remoteAddr, requestHost)
if err != nil {
failedOriginIds = append(failedOriginIds, origin.Id)
remotelogs.ServerError(serverId, "TCP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
SharedOriginStateManager.Fail(origin, requestHost, reverseProxy, func() {
reverseProxy.ResetScheduling()
})
continue
} else {
if !origin.IsOk {
SharedOriginStateManager.Success(origin, func() {
reverseProxy.ResetScheduling()
})
}
return
}
}
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
if err == nil {
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
}
return
}

View File

@@ -15,6 +15,10 @@ import (
"time"
)
const (
UDPConnLifeSeconds = 30
)
type UDPListener struct {
BaseListener
@@ -135,16 +139,39 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
var retries = 3
var addr string
var failedOriginIds []int64
for i := 0; i < retries; i++ {
var origin = reverseProxy.NextOrigin(nil)
var origin *serverconfigs.OriginConfig
if len(failedOriginIds) > 0 {
origin = reverseProxy.AnyOrigin(nil, failedOriginIds)
}
if origin == nil {
origin = reverseProxy.NextOrigin(nil)
}
if origin == nil {
continue
}
conn, addr, err = OriginConnect(origin, this.port, remoteAddr.String(), "")
if err != nil {
failedOriginIds = append(failedOriginIds, origin.Id)
remotelogs.ServerError(serverId, "UDP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
SharedOriginStateManager.Fail(origin, "", reverseProxy, func() {
reverseProxy.ResetScheduling()
})
continue
} else {
if !origin.IsOk {
SharedOriginStateManager.Success(origin, func() {
reverseProxy.ResetScheduling()
})
}
// PROXY Protocol
if reverseProxy != nil &&
reverseProxy.ProxyProtocol != nil &&
@@ -171,14 +198,17 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
return
}
}
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
if err == nil {
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
}
return
}
// 回收连接
func (this *UDPListener) gcConns() {
this.connLocker.Lock()
closingConns := []*UDPConn{}
var closingConns = []*UDPConn{}
for addr, conn := range this.connMap {
if !conn.IsOk() {
closingConns = append(closingConns, conn)
@@ -203,7 +233,7 @@ type UDPConn struct {
}
func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *net.UDPConn, serverConn *net.UDPConn) *UDPConn {
conn := &UDPConn{
var conn = &UDPConn{
addr: addr,
proxyConn: proxyConn,
serverConn: serverConn,
@@ -217,7 +247,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *ne
}
goman.New(func() {
buffer := utils.BytePool4k.Get()
var buffer = utils.BytePool4k.Get()
defer func() {
utils.BytePool4k.Put(buffer)
}()
@@ -232,9 +262,13 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *ne
break
}
// 记录流量
// 记录流量和带宽
if server != nil {
// 流量
stats.SharedTrafficStatManager.Add(server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
// 带宽
stats.SharedBandwidthStatManager.Add(server.UserId, server.Id, int64(n))
}
}
if err != nil {
@@ -268,5 +302,5 @@ func (this *UDPConn) IsOk() bool {
if !this.isOk {
return false
}
return time.Now().Unix()-this.activatedAt < 30 // 如果超过 N 秒没有活动我们认为是超时
return time.Now().Unix()-this.activatedAt < UDPConnLifeSeconds // 如果超过 N 秒没有活动我们认为是超时
}

View File

@@ -4,12 +4,14 @@ import (
"bytes"
"encoding/json"
"errors"
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/configs"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
@@ -21,6 +23,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/andybalholm/brotli"
"github.com/iwind/TeaGo/Tea"
@@ -30,7 +33,6 @@ import (
"github.com/iwind/TeaGo/types"
"github.com/iwind/gosock/pkg/gosock"
"gopkg.in/yaml.v3"
"io/ioutil"
"log"
"os"
"os/exec"
@@ -79,7 +81,7 @@ func (this *Node) Test() error {
if err != nil {
return errors.New("test rpc failed: " + err.Error())
}
_, err = rpcClient.APINodeRPC().FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
_, err = rpcClient.APINodeRPC.FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
if err != nil {
return errors.New("test rpc failed: " + err.Error())
}
@@ -102,9 +104,6 @@ func (this *Node) Start() {
// 监听signal
this.listenSignals()
// 启动事件
events.Notify(events.EventStart)
// 本地Sock
err := this.listenSock()
if err != nil {
@@ -112,9 +111,19 @@ func (this *Node) Start() {
return
}
// 启动IP库
remotelogs.Println("NODE", "initializing ip library ...")
err = iplib.InitDefault()
if err != nil {
remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
}
// 检查硬盘类型
this.checkDisk()
// 启动事件
events.Notify(events.EventStart)
// 读取API配置
remotelogs.Println("NODE", "init config ...")
err = this.syncConfig(0)
@@ -147,11 +156,21 @@ func (this *Node) Start() {
// 启动同步计时器
this.startSyncTimer()
// 状态变更计时器
// 更新IP库
goman.New(func() {
iplib.NewUpdater(NewIPLibraryUpdater(), 10*time.Minute).Start()
})
// 监控节点运行状态
goman.New(func() {
NewNodeStatusExecutor().Listen()
})
// 同步时间
goman.New(func() {
clock.Start()
})
// 读取配置
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
@@ -207,8 +226,9 @@ func (this *Node) Start() {
// Daemon 实现守护进程
func (this *Node) Daemon() {
isDebug := lists.ContainsString(os.Args, "debug")
isDebug = true
teaconst.IsDaemon = true
var isDebug = lists.ContainsString(os.Args, "debug")
for {
conn, err := this.sock.Dial()
if err != nil {
@@ -227,13 +247,18 @@ func (this *Node) Daemon() {
_ = os.Setenv("EdgeDaemon", "on")
_ = os.Setenv("EdgeBackground", "on")
cmd := exec.Command(exe)
var cmd = exec.Command(exe)
var buf = &bytes.Buffer{}
cmd.Stderr = buf
err = cmd.Start()
if err != nil {
return err
}
err = cmd.Wait()
if err != nil {
if isDebug {
log.Println("[DAEMON]" + buf.String())
}
return err
}
return nil
@@ -289,17 +314,25 @@ func (this *Node) loop() error {
}
var nodeCtx = rpcClient.Context()
tasksResp, err := rpcClient.NodeTaskRPC().FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
tasksResp, err := rpcClient.NodeTaskRPC.FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
if err != nil {
if rpc.IsConnError(err) && !Tea.IsTesting() {
return nil
}
return errors.New("read node tasks failed: " + err.Error())
}
for _, task := range tasksResp.NodeTasks {
switch task.Type {
case "ipItemChanged":
iplibrary.IPListUpdateNotify <- true
// 防止阻塞
select {
case iplibrary.IPListUpdateNotify <- true:
default:
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -318,13 +351,13 @@ func (this *Node) loop() error {
err = this.syncConfig(task.Version)
}
if err != nil {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: false,
Error: err.Error(),
})
} else {
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -333,11 +366,12 @@ func (this *Node) loop() error {
if err != nil {
return err
}
case "nodeVersionChanged":
goman.New(func() {
sharedUpgradeManager.Start()
})
if !sharedUpgradeManager.IsInstalling() {
goman.New(func() {
sharedUpgradeManager.Start()
})
}
case "scriptsChanged":
err = this.reloadCommonScripts()
if err != nil {
@@ -345,7 +379,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -354,7 +388,7 @@ func (this *Node) loop() error {
return err
}
case "nodeLevelChanged":
levelInfoResp, err := rpcClient.NodeRPC().FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
levelInfoResp, err := rpcClient.NodeRPC.FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
if err != nil {
return err
}
@@ -371,7 +405,7 @@ func (this *Node) loop() error {
sharedNodeConfig.ParentNodes = parentNodes
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -380,7 +414,7 @@ func (this *Node) loop() error {
return err
}
case "ddosProtectionChanged":
resp, err := rpcClient.NodeRPC().FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
resp, err := rpcClient.NodeRPC.FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
if err != nil {
return err
}
@@ -407,7 +441,7 @@ func (this *Node) loop() error {
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
@@ -452,7 +486,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
nodeCtx := rpcClient.Context()
// TODO 这里考虑只同步版本号有变更的
configResp, err := rpcClient.NodeRPC().FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
configResp, err := rpcClient.NodeRPC.FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
Version: -1, // 更新所有版本
Compress: true,
NodeTaskVersion: taskVersion,
@@ -543,7 +577,7 @@ func (this *Node) syncServerConfig(serverId int64) error {
if err != nil {
return err
}
resp, err := rpcClient.ServerRPC().ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
resp, err := rpcClient.ServerRPC.ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
if err != nil {
return err
}
@@ -605,7 +639,7 @@ func (this *Node) startSyncTimer() {
// 检查集群设置
func (this *Node) checkClusterConfig() error {
configFile := Tea.ConfigFile("cluster.yaml")
data, err := ioutil.ReadFile(configFile)
data, err := os.ReadFile(configFile)
if err != nil {
return err
}
@@ -625,7 +659,7 @@ func (this *Node) checkClusterConfig() error {
}
logs.Println("[NODE]registering node to cluster ...")
resp, err := rpcClient.NodeRPC().RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
resp, err := rpcClient.NodeRPC.RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
if err != nil {
return err
}
@@ -635,11 +669,13 @@ func (this *Node) checkClusterConfig() error {
if len(resp.Endpoints) == 0 {
resp.Endpoints = []string{}
}
apiConfig := &configs.APIConfig{
var apiConfig = &configs.APIConfig{
RPC: struct {
Endpoints []string `yaml:"endpoints"`
Endpoints []string `yaml:"endpoints"`
DisableUpdate bool `yaml:"disableUpdate"`
}{
Endpoints: resp.Endpoints,
Endpoints: resp.Endpoints,
DisableUpdate: false,
},
NodeId: resp.UniqueId,
Secret: resp.Secret,
@@ -673,7 +709,7 @@ func (this *Node) listenSock() error {
if this.sock.IsListening() {
reply, err := this.sock.Send(&gosock.Command{Code: "pid"})
if err == nil {
return errors.New("error: the process is already running, pid: " + maps.NewMap(reply.Params).GetString("pid"))
return errors.New("error: the process is already running, pid: " + types.String(maps.NewMap(reply.Params).GetInt("pid")))
} else {
return errors.New("error: the process is already running")
}
@@ -711,6 +747,7 @@ func (this *Node) listenSock() error {
_ = this.sock.Close()
events.Notify(events.EventQuit)
events.Notify(events.EventTerminated)
// 监控连接数如果连接数为0则退出进程
goman.New(func() {
@@ -761,20 +798,24 @@ func (this *Node) listenSock() error {
},
})
case "conns":
ipConns, serverConns := sharedClientConnLimiter.Conns()
var addrs = []string{}
var connMap = conns.SharedMap.AllConns()
for _, conn := range connMap {
addrs = append(addrs, conn.RemoteAddr().String())
}
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"ipConns": ipConns,
"serverConns": serverConns,
"total": sharedListenerManager.TotalActiveConnections(),
"addrs": addrs,
"total": len(addrs),
},
})
case "dropIP":
var m = maps.NewMap(cmd.Params)
var ip = m.GetString("ip")
var timeSeconds = m.GetInt("timeoutSeconds")
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds)
var async = m.GetBool("async")
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds, async)
if err != nil {
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
@@ -838,6 +879,11 @@ func (this *Node) listenSock() error {
} else {
_ = cmd.ReplyOk()
}
case "bandwidth":
var m = stats.SharedBandwidthStatManager.Map()
_ = cmd.Reply(&gosock.Command{Params: maps.Map{
"stats": m,
}})
}
})
@@ -848,7 +894,7 @@ func (this *Node) listenSock() error {
})
events.OnKey(events.EventQuit, this, func() {
logs.Println("NODE", "quit unix sock")
remotelogs.Println("NODE", "quit unix sock")
_ = this.sock.Close()
})
@@ -992,19 +1038,22 @@ func (this *Node) reloadServer() {
}
func (this *Node) checkDisk() {
if runtime.GOOS == "linux" {
if runtime.GOOS != "linux" {
return
}
for n := 'a'; n <= 'z'; n++ {
for _, path := range []string{
"/sys/block/vda/queue/rotational",
"/sys/block/sda/queue/rotational",
"/sys/block/vd" + string(n) + "/queue/rotational",
"/sys/block/sd" + string(n) + "/queue/rotational",
} {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
continue
}
if string(data) == "0" {
teaconst.DiskIsFast = true
}
break
return
}
}
}

View File

@@ -9,7 +9,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"os"
"syscall"
)
@@ -25,7 +24,7 @@ func (this *Node) handlePanic() {
var panicFile = Tea.Root + "/logs/panic.log"
// 分析panic
data, err := ioutil.ReadFile(panicFile)
data, err := os.ReadFile(panicFile)
if err == nil {
var index = bytes.Index(data, []byte("panic:"))
if index >= 0 {

View File

@@ -30,10 +30,14 @@ type NodeStatusExecutor struct {
cpuUpdatedTime time.Time
cpuLogicalCount int
cpuPhysicalCount int
ticker *time.Ticker
}
func NewNodeStatusExecutor() *NodeStatusExecutor {
return &NodeStatusExecutor{}
return &NodeStatusExecutor{
ticker: time.NewTicker(30 * time.Second),
}
}
func (this *NodeStatusExecutor) Listen() {
@@ -41,15 +45,12 @@ func (this *NodeStatusExecutor) Listen() {
this.cpuUpdatedTime = time.Now()
this.update()
// TODO 这个时间间隔可以配置
var ticker = time.NewTicker(30 * time.Second)
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("NODE_STATUS", "quit executor")
ticker.Stop()
this.ticker.Stop()
})
for range ticker.C {
for range this.ticker.C {
this.isFirstTime = false
this.update()
}
@@ -68,6 +69,8 @@ func (this *NodeStatusExecutor) update() {
status.BuildVersionCode = utils.VersionToLong(teaconst.Version)
status.OS = runtime.GOOS
status.Arch = runtime.GOARCH
exe, _ := os.Executable()
status.ExePath = exe
status.ConfigVersion = sharedNodeConfig.Version
status.IsActive = true
status.ConnectionCount = sharedListenerManager.TotalActiveConnections()
@@ -122,7 +125,7 @@ func (this *NodeStatusExecutor) update() {
remotelogs.Error("NODE_STATUS", "failed to open rpc: "+err.Error())
return
}
_, err = rpcClient.NodeRPC().UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
_, err = rpcClient.NodeRPC.UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
StatusJSON: jsonData,
})
if err != nil {

View File

@@ -11,7 +11,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/maps"
"io/ioutil"
"os"
"os/exec"
"runtime"
@@ -104,7 +103,7 @@ func (this *SystemServiceManager) setupSystemd(params maps.Map) error {
if output == "enabled" {
// 检查文件路径是否变化
data, err := ioutil.ReadFile("/etc/systemd/system/" + teaconst.SystemdServiceName + ".service")
data, err := os.ReadFile("/etc/systemd/system/" + teaconst.SystemdServiceName + ".service")
if err == nil && bytes.Index(data, []byte(exe)) > 0 {
return nil
}

View File

@@ -60,7 +60,7 @@ func (this *OCSPUpdateTask) Loop() error {
return err
}
resp, err := rpcClient.SSLCertRPC().ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
resp, err := rpcClient.SSLCertRPC.ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
Version: this.version,
Size: 100,
})

Some files were not shown because too many files have changed in this diff Show More