Compare commits

..

71 Commits

Author SHA1 Message Date
刘祥超
23a4b64e6d 优化代码 2022-07-17 17:08:10 +08:00
刘祥超
46a82f5988 升级compress package 2022-07-17 11:17:13 +08:00
刘祥超
1f37816a3a 改进MaxOpenFiles算法 2022-07-17 10:24:35 +08:00
刘祥超
2301e74b1c WAF策略增加记录区域封禁日志选项 2022-07-16 18:47:59 +08:00
刘祥超
a47d7d275c WAF策略增加记录请求Body选项 2022-07-16 17:05:37 +08:00
刘祥超
706519ac2e 优化代码 2022-07-16 14:48:57 +08:00
刘祥超
b8e193bc60 提升健康检查和UAM优先级 2022-07-16 09:49:26 +08:00
刘祥超
e44f9cc2fb cc2增加忽略常见文件扩展名选项 2022-07-15 12:02:19 +08:00
刘祥超
c6823ae3a8 优化连接代码/细化反向代理相关错误和警告提示 2022-07-15 11:15:55 +08:00
刘祥超
a18ebc0060 优化代码 2022-07-14 11:58:53 +08:00
刘祥超
7cf41ace47 缓存条件中启用客户端过期时间后,自动删除源站的Cache-Control Header 2022-07-14 11:03:34 +08:00
刘祥超
987350f0b4 升级Go版本为v1.18 2022-07-07 09:21:18 +08:00
刘祥超
ce7dda8cf5 增加服务带宽统计 2022-07-05 20:37:00 +08:00
刘祥超
af87cc9f16 增加UAM(5秒盾)集群设置 2022-07-03 22:09:37 +08:00
刘祥超
d5f6acf690 反向代理设置中增加移除回源主机名端口功能 2022-06-30 12:12:05 +08:00
刘祥超
92f541b0aa 实现源站端口跟随功能 2022-06-29 21:58:41 +08:00
刘祥超
fb6fee8c60 优化编译脚本 2022-06-29 15:42:50 +08:00
刘祥超
75fe0bd8c6 优化编译脚本 2022-06-29 14:51:13 +08:00
刘祥超
8f1f5f5bb4 支持ZSTD压缩 2022-06-27 22:40:36 +08:00
刘祥超
9fe6bc2dcc 限制源站错误检测最大并发数 2022-06-27 15:59:54 +08:00
刘祥超
b254cfc1a7 回源TLS/HTTPS携带ServerName信息 2022-06-27 12:01:33 +08:00
刘祥超
f8e155887f 找不到匹配的域名时自动记录日志、默认防cc攻击 2022-06-22 20:04:33 +08:00
刘祥超
c5a635d796 优化代码 2022-06-22 19:05:01 +08:00
刘祥超
607fa58ece 升级时备份可执行文件时将.old改成.dist,避免误解 2022-06-21 10:25:01 +08:00
刘祥超
0d5540295f 修复DDoS防护规则无法生成的Bug 2022-06-21 10:02:52 +08:00
刘祥超
9ce516caeb 修改版本号为v0.4.9 2022-06-20 16:00:44 +08:00
刘祥超
77bb1cf14e 版本更改为0.4.8.1 2022-06-20 09:34:06 +08:00
刘祥超
274284dbe1 静态文件分发也支持压缩、WebP转换 2022-06-19 11:39:21 +08:00
刘祥超
cd6d7221e8 不限制206 Partial Content两次写入文件的时间差 2022-06-18 20:05:09 +08:00
刘祥超
b1d0c8852e 如果缓存条件支持206 Partial Content,则第一次加载时自动尝试从分片缓存中读取内容 2022-06-18 19:31:10 +08:00
刘祥超
4c4033bb56 TCP负载均衡实现流量限制,达到限制后,关闭连接 2022-06-17 21:49:15 +08:00
刘祥超
6d642b75f6 延长预热超时时间 2022-06-16 20:32:11 +08:00
刘祥超
eb47e3a08c 删除缓存的时同时删除相关的缓存(压缩格式、WebP格式、http和https互换URL) 2022-06-15 12:54:56 +08:00
刘祥超
d82d16e28d 修复内容为空时无法缓存的Bug 2022-06-09 20:26:36 +08:00
刘祥超
b2fc785543 WAF规则中增加${requestURL}参数 2022-06-09 19:44:11 +08:00
刘祥超
189e3342ce 将缓存maxOpenFiles最小值从2改为4 2022-06-09 19:12:29 +08:00
刘祥超
885defbf31 优化nftables相关代码 2022-06-09 19:12:10 +08:00
刘祥超
74f1bf330d DNS解析库默认使用Go原生库 2022-06-07 11:49:38 +08:00
刘祥超
ad843d9d10 修复源站从http跳转到https导致无限循环的问题 2022-06-07 11:25:09 +08:00
刘祥超
13e718742d 节点状态上报时增加时间戳字段 2022-06-07 11:23:40 +08:00
刘祥超
771eff8fb1 增加刷新、预热缓存任务管理 2022-06-05 17:15:02 +08:00
刘祥超
20d7e0b1bf ACME申请证书时如果找不到Token,则直接跳过执行后面请求 2022-06-02 15:34:14 +08:00
刘祥超
e6c7bbec06 修复一个源站主备切换不灵敏的问题/WebSocket也支持源站主备自动切换 2022-05-23 20:01:26 +08:00
刘祥超
be61ef89fe fix typo 2022-05-23 16:15:02 +08:00
刘祥超
3d7d8f1e63 优化代码 2022-05-23 11:34:58 +08:00
刘祥超
a4fb465a19 在严格匹配域名模式下仍然可以通过节点IP进行健康检查 2022-05-23 11:17:53 +08:00
刘祥超
96c725c13d 增加LICENSE和README 2022-05-22 11:36:43 +08:00
刘祥超
7635def2fa 修正自动使用本地防火墙延长封禁时间逻辑 2022-05-21 22:15:11 +08:00
刘祥超
b704a73338 修复一个日志typo 2022-05-21 22:04:23 +08:00
刘祥超
123b5f5969 自动将同集群节点IP加入白名单/尝试使用本地防火墙提升黑名单连接封锁效率 2022-05-21 21:32:10 +08:00
刘祥超
eea2037444 优化验证码失败次数统计 2022-05-21 20:02:35 +08:00
刘祥超
4e6d2fa5ea WAF策略中增加验证码相关定制设置 2022-05-21 11:17:53 +08:00
刘祥超
14bb131e8d WAF CAPTCHA:刷新验证码页面也算入校验失败次数 2022-05-20 11:56:06 +08:00
刘祥超
31814bb54c 忽略301, 302, 303, 307, 308响应中没有Location的错误提示 2022-05-19 20:16:40 +08:00
刘祥超
49b8fd6e97 健康检查增加是否记录访问日志选项 2022-05-19 17:13:20 +08:00
刘祥超
a9d31a2e35 增加edge-node accesslog命令,用来在本地查看访问日志 2022-05-18 23:14:57 +08:00
刘祥超
298cef7f05 缩短指标统计队列长度 2022-05-18 21:41:34 +08:00
刘祥超
9bdd9a433c 实现基础的DDoS防护 2022-05-18 21:03:51 +08:00
刘祥超
45620dcdb7 计算CC的时候不再跨时间范围累积 2022-05-12 21:48:33 +08:00
刘祥超
84a5d38b0b 在启动时检查节点时间戳是否和API节点一致,如果不一致则上报 2022-05-12 21:07:45 +08:00
刘祥超
e812b3fcf6 X-Forwarded-For中包含当前客户端的IP 2022-05-08 16:56:10 +08:00
刘祥超
1bd16fa1d3 往硬盘刷数据时不统计maxOpenFiles 2022-05-07 22:02:41 +08:00
刘祥超
f3ea4957be fix typo 2022-05-05 11:01:03 +08:00
刘祥超
04da107c94 路由规则可以单独设置UAM(仅企业版可用) 2022-05-04 20:32:25 +08:00
刘祥超
e77de69a15 节点增加DNS解析库类型设置 2022-05-04 16:40:25 +08:00
刘祥超
e88eda56f5 自动替换Location中的地址时检查域名是否为当前域名 2022-05-04 10:48:14 +08:00
刘祥超
d6ceccc52e 增加基准测试 2022-05-01 10:40:19 +08:00
刘祥超
cd948ac68c 实现新的gzip库提升gzip性能 2022-04-30 22:22:30 +08:00
刘祥超
9eac8afa3d 停止节点时systemctl命令不阻塞当前进程 2022-04-26 12:22:00 +08:00
刘祥超
fb3610966a 白名单中的IP不受请求限制的影响 2022-04-25 11:11:25 +08:00
刘祥超
a6673449db 修改版本为0.4.8 2022-04-25 11:11:03 +08:00
174 changed files with 5535 additions and 1199 deletions

29
LICENSE Normal file
View File

@@ -0,0 +1,29 @@
BSD 3-Clause License
Copyright (c) 2020, LiuXiangChao
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

1
README.md Normal file
View File

@@ -0,0 +1 @@
GoEdge边缘节点源码

View File

@@ -3,7 +3,7 @@
function build() {
ROOT=$(dirname $0)
NAME="edge-node"
VERSION=$(lookup-version $ROOT/../internal/const/const.go)
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
DIST=$ROOT/"../dist/${NAME}"
MUSL_DIR="/usr/local/opt/musl-cross/bin"
GCC_X86_64_DIR="/usr/local/Cellar/x86_64-unknown-linux-gnu/10.3.0/bin"
@@ -13,21 +13,21 @@ function build() {
ARCH=${2}
TAG=${3}
if [ -z $OS ]; then
if [ -z "$OS" ]; then
echo "usage: build.sh OS ARCH"
exit
fi
if [ -z $ARCH ]; then
if [ -z "$ARCH" ]; then
echo "usage: build.sh OS ARCH"
exit
fi
if [ -z $TAG ]; then
if [ -z "$TAG" ]; then
TAG="community"
fi
echo "checking ..."
ZIP_PATH=$(which zip)
if [ -z $ZIP_PATH ]; then
if [ -z "$ZIP_PATH" ]; then
echo "we need 'zip' command to compress files"
exit
fi
@@ -36,28 +36,28 @@ function build() {
ZIP="${NAME}-${OS}-${ARCH}-${TAG}-v${VERSION}.zip"
echo "copying ..."
if [ ! -d $DIST ]; then
mkdir $DIST
mkdir $DIST/bin
mkdir $DIST/configs
mkdir $DIST/logs
mkdir $DIST/data
if [ ! -d "$DIST" ]; then
mkdir "$DIST"
mkdir "$DIST"/bin
mkdir "$DIST"/configs
mkdir "$DIST"/logs
mkdir "$DIST"/data
if [ "$TAG" = "plus" ]; then
mkdir $DIST/scripts
mkdir $DIST/scripts/js
mkdir "$DIST"/scripts
mkdir "$DIST"/scripts/js
fi
fi
cp $ROOT/configs/api.template.yaml $DIST/configs
cp -R $ROOT/www $DIST/
cp -R $ROOT/pages $DIST/
cp -R $ROOT/resources $DIST/
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
# we support TOA on linux/amd64 only
if [ $OS == "linux" -a $ARCH == "amd64" ]
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]
then
cp -R $ROOT/edge-toa $DIST
cp -R "$ROOT"/edge-toa "$DIST"
fi
echo "building ..."
@@ -112,14 +112,14 @@ function build() {
fi
fi
if [ ! -z $CC_PATH ]; then
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $BUILD_TAG -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
else
env GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 go build -trimpath -tags $TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
fi
# delete hidden files
find $DIST -name ".DS_Store" -delete
find $DIST -name ".gitignore" -delete
find "$DIST" -name ".DS_Store" -delete
find "$DIST" -name ".gitignore" -delete
echo "zip files"
cd "${DIST}/../" || exit
@@ -135,15 +135,15 @@ function build() {
function lookup-version() {
FILE=$1
VERSION_DATA=$(cat $FILE)
VERSION_DATA=$(cat "$FILE")
re="Version[ ]+=[ ]+\"([0-9.]+)\""
if [[ $VERSION_DATA =~ $re ]]; then
VERSION=${BASH_REMATCH[1]}
echo $VERSION
echo "$VERSION"
else
echo "could not match version"
exit
fi
}
build $1 $2 $3
build "$1" "$2" "$3"

View File

@@ -22,7 +22,7 @@ func main() {
app := apps.NewAppCmd().
Version(teaconst.Version).
Product(teaconst.ProductName).
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof]").
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog]").
Usage(teaconst.ProcessName + " [trackers|goman|conns|gc]").
Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove] IP")
@@ -258,6 +258,53 @@ func main() {
}
}
})
app.On("accesslog", func() {
// local sock
var tmpDir = os.TempDir()
var sockFile = tmpDir + "/" + teaconst.AccessLogSockName
_, err := os.Stat(sockFile)
if err != nil {
if !os.IsNotExist(err) {
fmt.Println("[ERROR]" + err.Error())
return
}
}
var processSock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := processSock.Send(&gosock.Command{
Code: "accesslog",
})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
if reply.Code == "error" {
var errString = maps.NewMap(reply.Params).GetString("error")
if len(errString) > 0 {
fmt.Println("[ERROR]" + errString)
return
}
}
conn, err := net.Dial("unix", sockFile)
if err != nil {
fmt.Println("[ERROR]start reading access log failed: " + err.Error())
return
}
defer func() {
_ = conn.Close()
}()
var buf = make([]byte, 1024)
for {
n, err := conn.Read(buf)
if n > 0 {
fmt.Print(string(buf[:n]))
}
if err != nil {
break
}
}
})
app.Run(func() {
node := nodes.NewNode()
node.Start()

3
dist/.gitignore vendored
View File

@@ -1 +1,2 @@
*.zip
*.zip
edge-node

37
go.mod
View File

@@ -1,15 +1,16 @@
module github.com/TeaOSLab/EdgeNode
go 1.15
go 1.18
replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
replace (
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
)
require (
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
github.com/andybalholm/brotli v1.0.4
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670
github.com/cespare/xxhash v1.1.0
github.com/chai2010/webp v1.1.0 // indirect
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
github.com/fsnotify/fsnotify v1.5.1
github.com/go-redis/redis/v8 v8.11.5
@@ -19,9 +20,9 @@ require (
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
github.com/kr/text v0.2.0 // indirect
github.com/klauspost/compress v1.15.8
github.com/mattn/go-sqlite3 v1.14.9
github.com/mdlayher/netlink v1.4.2
github.com/miekg/dns v1.1.43
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
@@ -31,6 +32,30 @@ require (
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/text v0.3.7
google.golang.org/grpc v1.45.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)
require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/webp v1.1.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/tklauser/go-sysconf v0.3.9 // indirect
github.com/tklauser/numcpus v0.3.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/tools v0.1.8 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
honnef.co/go/tools v0.2.2 // indirect
)

38
go.sum
View File

@@ -22,11 +22,7 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -50,12 +46,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
@@ -65,7 +56,6 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -96,17 +86,13 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
@@ -126,6 +112,10 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4h
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -167,22 +157,15 @@ github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRG
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ=
github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
@@ -257,7 +240,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -286,11 +268,9 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -301,7 +281,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -312,8 +291,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -322,14 +299,10 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -344,7 +317,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
@@ -400,7 +372,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -409,4 +380,3 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
rogchap.com/v8go v0.7.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs=

View File

@@ -217,7 +217,10 @@ func (this *AppCmd) runStop() {
if runtime.GOOS == "linux" {
systemctl, _ := exec.LookPath("systemctl")
if len(systemctl) > 0 {
_ = exec.Command(systemctl, "stop", teaconst.SystemdServiceName).Run()
go func() {
// 有可能会长时间执行,这里不阻塞进程
_ = exec.Command(systemctl, "stop", teaconst.SystemdServiceName).Run()
}()
}
}

View File

@@ -214,3 +214,15 @@ func (this *Manager) FindAllCachePaths() []string {
}
return result
}
// FindAllStorages 读取所有缓存存储
func (this *Manager) FindAllStorages() []StorageInterface {
this.locker.Lock()
defer this.locker.Unlock()
var storages = []StorageInterface{}
for _, storage := range this.storageMap {
storages = append(storages, storage)
}
return storages
}

View File

@@ -3,44 +3,24 @@
package caches
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"sync/atomic"
"time"
)
const (
minOpenFilesValue int32 = 2
maxOpenFilesValue int32 = 65535
modeSlow int32 = 1
modeFast int32 = 2
)
// MaxOpenFiles max open files manager
type MaxOpenFiles struct {
step int32
maxOpenFiles int32
ptr *int32
ticker *time.Ticker
mode int32
lastOpens int32
currentOpens int32
ticker *time.Ticker
mode int32
}
func NewMaxOpenFiles(step int32) *MaxOpenFiles {
if step <= 0 {
step = 2
}
var f = &MaxOpenFiles{
step: step,
maxOpenFiles: 2,
}
if teaconst.DiskIsFast {
f.maxOpenFiles = 32
}
f.ptr = &f.maxOpenFiles
func NewMaxOpenFiles() *MaxOpenFiles {
var f = &MaxOpenFiles{}
f.ticker = time.NewTicker(1 * time.Second)
f.init()
return f
@@ -49,50 +29,24 @@ func NewMaxOpenFiles(step int32) *MaxOpenFiles {
func (this *MaxOpenFiles) init() {
goman.New(func() {
for range this.ticker.C {
var mod = atomic.LoadInt32(&this.mode)
switch mod {
case modeSlow:
// we decrease more quickly, with more steps
if atomic.AddInt32(this.ptr, -this.step*2) <= 0 {
atomic.StoreInt32(this.ptr, minOpenFilesValue)
}
case modeFast:
// we increase only when file opens increases
var currentOpens = atomic.LoadInt32(&this.currentOpens)
if currentOpens > this.lastOpens {
if atomic.AddInt32(this.ptr, this.step) >= maxOpenFilesValue {
atomic.StoreInt32(this.ptr, maxOpenFilesValue)
}
}
this.lastOpens = currentOpens
atomic.StoreInt32(&this.currentOpens, 0)
}
// reset mod
atomic.StoreInt32(&this.mode, 0)
// reset mode
atomic.StoreInt32(&this.mode, modeFast)
}
})
}
func (this *MaxOpenFiles) Fast() {
if atomic.LoadInt32(&this.mode) == 0 {
this.mode = modeFast
}
atomic.AddInt32(&this.currentOpens, 1)
atomic.AddInt32(&this.mode, modeFast)
}
func (this *MaxOpenFiles) FinishAll() {
this.Fast()
}
func (this *MaxOpenFiles) Slow() {
atomic.StoreInt32(&this.mode, modeSlow)
}
func (this *MaxOpenFiles) Max() int32 {
if atomic.LoadInt32(&this.mode) == modeSlow {
return 0
}
var v = atomic.LoadInt32(this.ptr)
if v <= minOpenFilesValue {
return minOpenFilesValue
}
return v
func (this *MaxOpenFiles) Next() bool {
return atomic.LoadInt32(&this.mode) != modeSlow
}

View File

@@ -9,20 +9,27 @@ import (
)
func TestNewMaxOpenFiles(t *testing.T) {
var maxOpenFiles = caches.NewMaxOpenFiles(2)
var maxOpenFiles = caches.NewMaxOpenFiles()
maxOpenFiles.Fast()
t.Log(maxOpenFiles.Max())
t.Log("fast:", maxOpenFiles.Next())
maxOpenFiles.Slow()
t.Log("slow:", maxOpenFiles.Next())
time.Sleep(1*time.Second + 1*time.Millisecond)
t.Log("slow 1 second:", maxOpenFiles.Next())
maxOpenFiles.Slow()
t.Log("slow:", maxOpenFiles.Next())
maxOpenFiles.Slow()
t.Log("slow:", maxOpenFiles.Next())
maxOpenFiles.Fast()
time.Sleep(1 * time.Second)
t.Log(maxOpenFiles.Max())
t.Log("slow 1 second:", maxOpenFiles.Next())
maxOpenFiles.Slow()
t.Log(maxOpenFiles.Max())
t.Log("slow:", maxOpenFiles.Next())
maxOpenFiles.Slow()
t.Log(maxOpenFiles.Max())
maxOpenFiles.Slow()
t.Log(maxOpenFiles.Max())
maxOpenFiles.Fast()
t.Log("fast:", maxOpenFiles.Next())
}

View File

@@ -63,7 +63,7 @@ const (
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
var sharedWritingFileKeyLocker = sync.Mutex{}
var maxOpenFiles = NewMaxOpenFiles(2)
var maxOpenFiles = NewMaxOpenFiles()
const maxOpenFilesSlowCost = 500 * time.Microsecond // 0.5ms
@@ -428,7 +428,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
return nil, ErrFileIsWriting
}
if len(sharedWritingFileKeyMap) >= int(maxOpenFiles.Max()) {
if !isFlushing && !maxOpenFiles.Next() {
sharedWritingFileKeyLocker.Unlock()
return nil, ErrTooManyOpenFiles
}
@@ -439,6 +439,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
if !isOk {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
if len(sharedWritingFileKeyMap) == 0 {
maxOpenFiles.FinishAll()
}
sharedWritingFileKeyLocker.Unlock()
}
}()
@@ -481,11 +484,16 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
openFileCache.Close(cachePath)
}
// 查询当前已有缓存文件
stat, err := os.Stat(cachePath)
if err == nil && time.Now().Sub(stat.ModTime()) <= 1*time.Second {
// 检查两次写入缓存的时间是否过于相近,分片内容不受此限制
if err == nil && !isPartial && time.Now().Sub(stat.ModTime()) <= 1*time.Second {
// 防止并发连续写入
return nil, ErrFileIsWriting
}
// 构造文件名
var tmpPath = cachePath
var existsFile = false
if stat != nil {
@@ -534,10 +542,12 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
if err != nil {
return nil, err
}
if time.Since(before) >= maxOpenFilesSlowCost {
maxOpenFiles.Slow()
} else {
maxOpenFiles.Fast()
if !isFlushing {
if time.Since(before) >= maxOpenFilesSlowCost {
maxOpenFiles.Slow()
} else {
maxOpenFiles.Fast()
}
}
var removeOnFailure = true
@@ -601,12 +611,18 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
return NewPartialFileWriter(writer, key, expiredAt, isNewCreated, isPartial, partialBodyOffset, ranges, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
if len(sharedWritingFileKeyMap) == 0 {
maxOpenFiles.FinishAll()
}
sharedWritingFileKeyLocker.Unlock()
}), nil
} else {
return NewFileWriter(this, writer, key, expiredAt, -1, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
if len(sharedWritingFileKeyMap) == 0 {
maxOpenFiles.FinishAll()
}
sharedWritingFileKeyLocker.Unlock()
}), nil
}
@@ -767,9 +783,10 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
return err
}
}
return nil
}
// 文件
// URL
for _, key := range keys {
hash, path := this.keyPath(key)
err := this.removeCacheFile(path)

View File

@@ -35,6 +35,7 @@ type StorageInterface interface {
CleanAll() error
// Purge 批量删除缓存
// urlType 值为file|dir
Purge(keys []string, urlType string) error
// Stop 停止缓存策略

View File

@@ -267,8 +267,10 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
return err
}
}
return nil
}
// URL
for _, key := range keys {
err := this.Delete(key)
if err != nil {

View File

@@ -3,7 +3,7 @@
package compressions
import (
"compress/gzip"
"github.com/klauspost/compress/gzip"
"io"
)

View File

@@ -0,0 +1,20 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
var sharedZSTDReaderPool *ReaderPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedZSTDReaderPool = NewReaderPool(maxSize, func(reader io.Reader) (Reader, error) {
return newZSTDReader(reader)
})
}

View File

@@ -0,0 +1,45 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/klauspost/compress/zstd"
"io"
)
type ZSTDReader struct {
BaseReader
reader *zstd.Decoder
}
func NewZSTDReader(reader io.Reader) (Reader, error) {
return sharedZSTDReaderPool.Get(reader)
}
func newZSTDReader(reader io.Reader) (Reader, error) {
r, err := zstd.NewReader(reader)
if err != nil {
return nil, err
}
return &ZSTDReader{
reader: r,
}, nil
}
func (this *ZSTDReader) Read(p []byte) (n int, err error) {
return this.reader.Read(p)
}
func (this *ZSTDReader) Reset(reader io.Reader) error {
return this.reader.Reset(reader)
}
func (this *ZSTDReader) RawClose() error {
this.reader.Close()
return nil
}
func (this *ZSTDReader) Close() error {
return this.Finish(this)
}

View File

@@ -0,0 +1,106 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"io"
"strings"
"testing"
)
func TestZSTDReader(t *testing.T) {
for _, testString := range []string{"Hello", "World", "Ni", "Hao"} {
t.Log("===", testString, "===")
var buf = &bytes.Buffer{}
writer, err := compressions.NewZSTDWriter(buf, 5)
if err != nil {
t.Fatal(err)
}
_, err = writer.Write([]byte(testString))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := compressions.NewZSTDReader(buf)
if err != nil {
t.Fatal(err)
}
var data = make([]byte, 4096)
for {
n, err := reader.Read(data)
if n > 0 {
t.Log(string(data[:n]))
}
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
}
func BenchmarkZSTDReader(b *testing.B) {
var randomData = func() []byte {
var b = strings.Builder{}
for i := 0; i < 1024; i++ {
b.WriteString(types.String(rands.Int64() % 10))
}
return []byte(b.String())
}
var buf = &bytes.Buffer{}
writer, err := compressions.NewZSTDWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
_, err = writer.Write(randomData())
if err != nil {
b.Fatal(err)
}
err = writer.Close()
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
var newBytes = make([]byte, buf.Len())
copy(newBytes, buf.Bytes())
reader, err := compressions.NewZSTDReader(bytes.NewReader(newBytes))
if err != nil {
b.Fatal(err)
}
var data = make([]byte, 4096)
for {
n, err := reader.Read(data)
if n > 0 {
_ = data[:n]
}
if err != nil {
if err == io.EOF {
break
}
b.Fatal(err)
}
}
err = reader.Close()
if err != nil {
b.Fatal(err)
}
}
}

View File

@@ -14,13 +14,19 @@ const (
ContentEncodingBr ContentEncoding = "br"
ContentEncodingGzip ContentEncoding = "gzip"
ContentEncodingDeflate ContentEncoding = "deflate"
ContentEncodingZSTD ContentEncoding = "zstd"
)
var ErrNotSupportedContentEncoding = errors.New("not supported content encoding")
// AllEncodings 当前支持的所有编码
func AllEncodings() []ContentEncoding {
return []ContentEncoding{ContentEncodingBr, ContentEncodingGzip, ContentEncodingDeflate}
return []ContentEncoding{
ContentEncodingBr,
ContentEncodingGzip,
ContentEncodingZSTD,
ContentEncodingDeflate,
}
}
// NewReader 获取Reader
@@ -32,6 +38,8 @@ func NewReader(reader io.Reader, contentEncoding ContentEncoding) (Reader, error
return NewGzipReader(reader)
case ContentEncodingDeflate:
return NewDeflateReader(reader)
case ContentEncodingZSTD:
return NewZSTDReader(reader)
}
return nil, ErrNotSupportedContentEncoding
}
@@ -45,6 +53,8 @@ func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType,
return NewDeflateWriter(writer, level)
case serverconfigs.HTTPCompressionTypeBrotli:
return NewBrotliWriter(writer, level)
case serverconfigs.HTTPCompressionTypeZSTD:
return NewZSTDWriter(writer, level)
}
return nil, errors.New("invalid compression type '" + compressType + "'")
}

View File

@@ -3,7 +3,7 @@
package compressions
import (
"compress/gzip"
"github.com/klauspost/compress/gzip"
"io"
)

View File

@@ -34,3 +34,31 @@ func BenchmarkGzipWriter_Write(b *testing.B) {
_ = writer.Close()
}
}
func BenchmarkGzipWriter_Write_Parallel(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var buf = &bytes.Buffer{}
writer, err := compressions.NewGzipWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
})
}

View File

@@ -0,0 +1,21 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/klauspost/compress/zstd"
"io"
)
var sharedZSTDWriterPool *WriterPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedZSTDWriterPool = NewWriterPool(maxSize, int(zstd.SpeedBestCompression), func(writer io.Writer, level int) (Writer, error) {
return newZSTDWriter(writer, level)
})
}

View File

@@ -0,0 +1,57 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/klauspost/compress/zstd"
"io"
)
type ZSTDWriter struct {
BaseWriter
writer *zstd.Encoder
level int
}
func NewZSTDWriter(writer io.Writer, level int) (Writer, error) {
return sharedZSTDWriterPool.Get(writer, level)
}
func newZSTDWriter(writer io.Writer, level int) (Writer, error) {
var zstdLevel = zstd.EncoderLevelFromZstd(level)
zstdWriter, err := zstd.NewWriter(writer, zstd.WithEncoderLevel(zstdLevel))
if err != nil {
return nil, err
}
return &ZSTDWriter{
writer: zstdWriter,
level: level,
}, nil
}
func (this *ZSTDWriter) Write(p []byte) (int, error) {
return this.writer.Write(p)
}
func (this *ZSTDWriter) Flush() error {
return this.writer.Flush()
}
func (this *ZSTDWriter) Reset(writer io.Writer) {
this.writer.Reset(writer)
}
func (this *ZSTDWriter) RawClose() error {
return this.writer.Close()
}
func (this *ZSTDWriter) Close() error {
return this.Finish(this)
}
func (this *ZSTDWriter) Level() int {
return this.level
}

View File

@@ -0,0 +1,82 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"strings"
"testing"
)
func TestNewZSTDWriter(t *testing.T) {
var buf = &bytes.Buffer{}
writer, err := compressions.NewZSTDWriter(buf, 10)
if err != nil {
t.Fatal(err)
}
var originData = []byte(strings.Repeat("Hello", 1024))
_, err = writer.Write(originData)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log("origin data:", len(originData), "result:", buf.Len())
}
func BenchmarkZSTDWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
for i := 0; i < b.N; i++ {
var buf = &bytes.Buffer{}
writer, err := compressions.NewZSTDWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
}
func BenchmarkZSTDWriter_Write_Parallel(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var buf = &bytes.Buffer{}
writer, err := compressions.NewZSTDWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
})
}

View File

@@ -1,6 +1,6 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build community
// +build community
//go:build !plus
// +build !plus
package teaconst

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.4.7"
Version = "0.4.9"
ProductName = "Edge Node"
ProcessName = "edge-node"
@@ -13,4 +13,6 @@ const (
// SystemdServiceName systemd
SystemdServiceName = "edge-node"
AccessLogSockName = "edge-node.accesslog.sock"
)

View File

@@ -3,9 +3,10 @@ package events
type Event = string
const (
EventStart Event = "start" // start loading
EventLoaded Event = "loaded" // first load
EventQuit Event = "quit" // quit node gracefully
EventReload Event = "reload" // reload config
EventTerminated Event = "terminated" // process terminated
EventStart Event = "start" // start loading
EventLoaded Event = "loaded" // first load
EventQuit Event = "quit" // quit node gracefully
EventReload Event = "reload" // reload config
EventTerminated Event = "terminated" // process terminated
EventNFTablesReady Event = "nftablesReady" // nftables ready
)

View File

@@ -1 +0,0 @@
firewall_nftables_test.go

View File

@@ -0,0 +1,502 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package firewalls
import (
"bytes"
"encoding/json"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"net"
"os/exec"
"strings"
)
var SharedDDoSProtectionManager = NewDDoSProtectionManager()
func init() {
events.On(events.EventReload, func() {
if nftablesInstance == nil {
return
}
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil {
err := SharedDDoSProtectionManager.Apply(nodeConfig.DDoSProtection)
if err != nil {
remotelogs.Error("FIREWALL", "apply DDoS protection failed: "+err.Error())
}
}
})
events.On(events.EventNFTablesReady, func() {
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil {
err := SharedDDoSProtectionManager.Apply(nodeConfig.DDoSProtection)
if err != nil {
remotelogs.Error("FIREWALL", "apply DDoS protection failed: "+err.Error())
}
}
})
}
// DDoSProtectionManager DDoS防护
type DDoSProtectionManager struct {
nftPath string
lastAllowIPList []string
lastConfig []byte
}
// NewDDoSProtectionManager 获取新对象
func NewDDoSProtectionManager() *DDoSProtectionManager {
nftPath, _ := exec.LookPath("nft")
return &DDoSProtectionManager{
nftPath: nftPath,
}
}
// Apply 应用配置
func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) error {
// 同集群节点IP白名单
var allowIPListChanged = false
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil {
var allowIPList = nodeConfig.AllowedIPs
if !utils.ContainsSameStrings(allowIPList, this.lastAllowIPList) {
allowIPListChanged = true
this.lastAllowIPList = allowIPList
}
}
// 对比配置
configJSON, err := json.Marshal(config)
if err != nil {
return errors.New("encode config to json failed: " + err.Error())
}
if !allowIPListChanged && bytes.Equal(this.lastConfig, configJSON) {
return nil
}
remotelogs.Println("FIREWALL", "change DDoS protection config")
if len(this.nftPath) == 0 {
return errors.New("can not find nft command")
}
if nftablesInstance == nil {
return errors.New("nftables instance should not be nil")
}
if config == nil {
// TCP
err := this.removeTCPRules()
if err != nil {
return err
}
// TODO other protocols
return nil
}
// TCP
if config.TCP == nil {
err := this.removeTCPRules()
if err != nil {
return err
}
} else {
// allow ip list
var allowIPList = []string{}
for _, ipConfig := range config.TCP.AllowIPList {
allowIPList = append(allowIPList, ipConfig.IP)
}
for _, ip := range this.lastAllowIPList {
if !lists.ContainsString(allowIPList, ip) {
allowIPList = append(allowIPList, ip)
}
}
err = this.updateAllowIPList(allowIPList)
if err != nil {
return err
}
// tcp
if config.TCP.IsOn {
err := this.addTCPRules(config.TCP)
if err != nil {
return err
}
} else {
err := this.removeTCPRules()
if err != nil {
return err
}
}
}
this.lastConfig = configJSON
return nil
}
// 添加TCP规则
func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig) error {
// 检查nft版本不能小于0.9
if len(nftablesInstance.version) > 0 && stringutil.VersionCompare("0.9", nftablesInstance.version) > 0 {
return nil
}
var ports = []int32{}
for _, portConfig := range tcpConfig.Ports {
if !lists.ContainsInt32(ports, portConfig.Port) {
ports = append(ports, portConfig.Port)
}
}
if len(ports) == 0 {
ports = []int32{80, 443}
}
for _, filter := range nftablesFilters {
chain, oldRules, err := this.getRules(filter)
if err != nil {
return errors.New("get old rules failed: " + err.Error())
}
var protocol = filter.protocol()
// max connections
var maxConnections = tcpConfig.MaxConnections
if maxConnections <= 0 {
maxConnections = nodeconfigs.DefaultTCPMaxConnections
if maxConnections <= 0 {
maxConnections = 100000
}
}
// max connections per ip
var maxConnectionsPerIP = tcpConfig.MaxConnectionsPerIP
if maxConnectionsPerIP <= 0 {
maxConnectionsPerIP = nodeconfigs.DefaultTCPMaxConnectionsPerIP
if maxConnectionsPerIP <= 0 {
maxConnectionsPerIP = 100000
}
}
// new connections rate
var newConnectionsRate = tcpConfig.NewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = 100000
}
}
// 检查是否有变化
var hasChanges = false
for _, port := range ports {
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "maxConnections", types.String(maxConnections)}) {
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}) {
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
hasChanges = true
break
}
}
if !hasChanges {
// 检查是否有多余的端口
var oldPorts = this.getTCPPorts(oldRules)
if !this.eqPorts(ports, oldPorts) {
hasChanges = true
}
}
if !hasChanges {
return nil
}
// 先清空所有相关规则
err = this.removeOldTCPRules(chain, oldRules)
if err != nil {
return errors.New("delete old rules failed: " + err.Error())
}
// 添加新规则
for _, port := range ports {
if maxConnections > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "count", "over", types.String(maxConnections), "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnections", types.String(maxConnections)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
if maxConnectionsPerIP > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
if newConnectionsRate > 0 {
// TODO 思考是否有惩罚机制
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
}
}
}
}
return nil
}
// 删除TCP规则
func (this *DDoSProtectionManager) removeTCPRules() error {
for _, filter := range nftablesFilters {
chain, rules, err := this.getRules(filter)
// TCP
err = this.removeOldTCPRules(chain, rules)
if err != nil {
return err
}
}
return nil
}
// 组合user data
// 数据中不能包含字母、数字、下划线以外的数据
func (this *DDoSProtectionManager) encodeUserData(attrs []string) string {
if attrs == nil {
return ""
}
return "ZZ" + strings.Join(attrs, "_") + "ZZ"
}
// 解码user data
func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
if len(data) == 0 {
return nil
}
var dataCopy = make([]byte, len(data))
copy(dataCopy, data)
var separatorLen = 2
var index1 = bytes.Index(dataCopy, []byte{'Z', 'Z'})
if index1 < 0 {
return nil
}
dataCopy = dataCopy[index1+separatorLen:]
var index2 = bytes.LastIndex(dataCopy, []byte{'Z', 'Z'})
if index2 < 0 {
return nil
}
var s = string(dataCopy[:index2])
var pieces = strings.Split(s, "_")
for index, piece := range pieces {
pieces[index] = strings.TrimSpace(piece)
}
return pieces
}
// 清除规则
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
for _, rule := range rules {
var pieces = this.decodeUserData(rule.UserData())
if len(pieces) != 4 {
continue
}
if pieces[0] != "tcp" {
continue
}
switch pieces[2] {
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
err := chain.DeleteRule(rule)
if err != nil {
return err
}
}
}
return nil
}
// 根据参数检查规则是否存在
func (this *DDoSProtectionManager) existsRule(rules []*nftables.Rule, attrs []string) (exists bool) {
if len(attrs) == 0 {
return false
}
for _, oldRule := range rules {
var pieces = this.decodeUserData(oldRule.UserData())
if len(attrs) != len(pieces) {
continue
}
var isSame = true
for index, piece := range pieces {
if strings.TrimSpace(piece) != attrs[index] {
isSame = false
break
}
}
if isSame {
return true
}
}
return false
}
// 获取规则中的端口号
func (this *DDoSProtectionManager) getTCPPorts(rules []*nftables.Rule) []int32 {
var ports = []int32{}
for _, rule := range rules {
var pieces = this.decodeUserData(rule.UserData())
if len(pieces) != 4 {
continue
}
if pieces[0] != "tcp" {
continue
}
var port = types.Int32(pieces[1])
if port > 0 && !lists.ContainsInt32(ports, port) {
ports = append(ports, port)
}
}
return ports
}
// 检查端口是否一样
func (this *DDoSProtectionManager) eqPorts(ports1 []int32, ports2 []int32) bool {
if len(ports1) != len(ports2) {
return false
}
var portMap = map[int32]bool{}
for _, port := range ports2 {
portMap[port] = true
}
for _, port := range ports1 {
_, ok := portMap[port]
if !ok {
return false
}
}
return true
}
// 查找Table
func (this *DDoSProtectionManager) getTable(filter *nftablesTableDefinition) (*nftables.Table, error) {
var family nftables.TableFamily
if filter.IsIPv4 {
family = nftables.TableFamilyIPv4
} else if filter.IsIPv6 {
family = nftables.TableFamilyIPv6
} else {
return nil, errors.New("table '" + filter.Name + "' should be IPv4 or IPv6")
}
return nftablesInstance.conn.GetTable(filter.Name, family)
}
// 查找所有规则
func (this *DDoSProtectionManager) getRules(filter *nftablesTableDefinition) (*nftables.Chain, []*nftables.Rule, error) {
table, err := this.getTable(filter)
if err != nil {
return nil, nil, errors.New("get table failed: " + err.Error())
}
chain, err := table.GetChain(nftablesChainName)
if err != nil {
return nil, nil, errors.New("get chain failed: " + err.Error())
}
rules, err := chain.GetRules()
return chain, rules, err
}
// 更新白名单
func (this *DDoSProtectionManager) updateAllowIPList(allIPList []string) error {
if nftablesInstance == nil {
return nil
}
var allMap = map[string]zero.Zero{}
for _, ip := range allIPList {
allMap[ip] = zero.New()
}
for _, set := range []*nftables.Set{nftablesInstance.allowIPv4Set, nftablesInstance.allowIPv6Set} {
var isIPv4 = set == nftablesInstance.allowIPv4Set
var isIPv6 = !isIPv4
// 现有的
oldList, err := set.GetIPElements()
if err != nil {
return err
}
var oldMap = map[string]zero.Zero{} // ip=> zero
for _, ip := range oldList {
oldMap[ip] = zero.New()
if (utils.IsIPv4(ip) && isIPv4) || (utils.IsIPv6(ip) && isIPv6) {
_, ok := allMap[ip]
if !ok {
// 不存在则删除
err = set.DeleteIPElement(ip)
if err != nil {
return errors.New("delete ip element '" + ip + "' failed: " + err.Error())
}
}
}
}
// 新增的
for _, ip := range allIPList {
var ipObj = net.ParseIP(ip)
if ipObj == nil {
continue
}
if (utils.IsIPv4(ip) && isIPv4) || (utils.IsIPv6(ip) && isIPv6) {
_, ok := oldMap[ip]
if !ok {
// 不存在则添加
err = set.AddIPElement(ip, nil)
if err != nil {
return errors.New("add ip '" + ip + "' failed: " + err.Error())
}
}
}
}
}
return nil
}

View File

@@ -0,0 +1,23 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build !linux
// +build !linux
package firewalls
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
)
var SharedDDoSProtectionManager = NewDDoSProtectionManager()
type DDoSProtectionManager struct {
nftPath string
}
func NewDDoSProtectionManager() *DDoSProtectionManager {
return &DDoSProtectionManager{}
}
func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) error {
return nil
}

View File

@@ -1,6 +1,4 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build !plus
// +build !plus
package firewalls
@@ -8,9 +6,11 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"runtime"
"sync"
)
var currentFirewall FirewallInterface
var firewallLocker = &sync.Mutex{}
// 初始化
func init() {
@@ -24,10 +24,28 @@ func init() {
// Firewall 查找当前系统中最适合的防火墙
func Firewall() FirewallInterface {
firewallLocker.Lock()
defer firewallLocker.Unlock()
if currentFirewall != nil {
return currentFirewall
}
// nftables
if runtime.GOOS == "linux" {
nftables, err := NewNFTablesFirewall()
if err != nil {
remotelogs.Warn("FIREWALL", "'nftables' should be installed on the system to enhance security (init failed: "+err.Error()+")")
} else {
if nftables.IsReady() {
currentFirewall = nftables
events.Notify(events.EventNFTablesReady)
return nftables
} else {
remotelogs.Warn("FIREWALL", "'nftables' should be enabled on the system to enhance security")
}
}
}
// firewalld
if runtime.GOOS == "linux" {
var firewalld = NewFirewalld()

View File

@@ -23,12 +23,13 @@ func NewFirewalld() *Firewalld {
path, err := exec.LookPath("firewall-cmd")
if err == nil && len(path) > 0 {
var cmd = exec.Command(path, "-V")
var cmd = exec.Command(path, "--state")
err := cmd.Run()
if err == nil {
firewalld.exe = path
// TODO check firewalld status with 'firewall-cmd --state' (running or not running),
// but we should recover the state when firewalld state changes, maybe check it every minutes
firewalld.isReady = true
firewalld.init()
}

View File

@@ -0,0 +1,395 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package firewalls
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
"net"
"os/exec"
"regexp"
"runtime"
"strings"
"time"
)
// check nft status, if being enabled we load it automatically
func init() {
if runtime.GOOS == "linux" {
var ticker = time.NewTicker(3 * time.Minute)
go func() {
for range ticker.C {
// if already ready, we break
if nftablesIsReady {
ticker.Stop()
break
}
_, err := exec.LookPath("nft")
if err == nil {
nftablesFirewall, err := NewNFTablesFirewall()
if err != nil {
continue
}
currentFirewall = nftablesFirewall
remotelogs.Println("FIREWALL", "nftables is ready")
// fire event
if nftablesFirewall.IsReady() {
events.Notify(events.EventNFTablesReady)
}
ticker.Stop()
break
}
}
}()
}
}
var nftablesInstance *NFTablesFirewall
var nftablesIsReady = false
var nftablesFilters = []*nftablesTableDefinition{
// we shorten the name for table name length restriction
{Name: "edge_dft_v4", IsIPv4: true},
{Name: "edge_dft_v6", IsIPv6: true},
}
var nftablesChainName = "input"
type nftablesTableDefinition struct {
Name string
IsIPv4 bool
IsIPv6 bool
}
func (this *nftablesTableDefinition) protocol() string {
if this.IsIPv6 {
return "ip6"
}
return "ip"
}
func NewNFTablesFirewall() (*NFTablesFirewall, error) {
var firewall = &NFTablesFirewall{
conn: nftables.NewConn(),
}
err := firewall.init()
if err != nil {
return nil, err
}
return firewall, nil
}
type NFTablesFirewall struct {
conn *nftables.Conn
isReady bool
version string
allowIPv4Set *nftables.Set
allowIPv6Set *nftables.Set
denyIPv4Set *nftables.Set
denyIPv6Set *nftables.Set
firewalld *Firewalld
}
func (this *NFTablesFirewall) init() error {
// check nft
nftPath, err := exec.LookPath("nft")
if err != nil {
return errors.New("nft not found")
}
this.version = this.readVersion(nftPath)
// table
for _, tableDef := range nftablesFilters {
var family nftables.TableFamily
if tableDef.IsIPv4 {
family = nftables.TableFamilyIPv4
} else if tableDef.IsIPv6 {
family = nftables.TableFamilyIPv6
} else {
return errors.New("invalid table family: " + types.String(tableDef))
}
table, err := this.conn.GetTable(tableDef.Name, family)
if err != nil {
if nftables.IsNotFound(err) {
if tableDef.IsIPv4 {
table, err = this.conn.AddIPv4Table(tableDef.Name)
} else if tableDef.IsIPv6 {
table, err = this.conn.AddIPv6Table(tableDef.Name)
}
if err != nil {
return errors.New("create table '" + tableDef.Name + "' failed: " + err.Error())
}
} else {
return errors.New("get table '" + tableDef.Name + "' failed: " + err.Error())
}
}
if table == nil {
return errors.New("can not create table '" + tableDef.Name + "'")
}
// chain
var chainName = nftablesChainName
chain, err := table.GetChain(chainName)
if err != nil {
if nftables.IsNotFound(err) {
chain, err = table.AddAcceptChain(chainName)
if err != nil {
return errors.New("create chain '" + chainName + "' failed: " + err.Error())
}
} else {
return errors.New("get chain '" + chainName + "' failed: " + err.Error())
}
}
if chain == nil {
return errors.New("can not create chain '" + chainName + "'")
}
// allow lo
var loRuleName = []byte("lo")
_, err = chain.GetRuleWithUserData(loRuleName)
if err != nil {
if nftables.IsNotFound(err) {
_, err = chain.AddAcceptInterfaceRule("lo", loRuleName)
}
if err != nil {
return errors.New("add 'lo' rule failed: " + err.Error())
}
}
// allow set
// "allow" should be always first
for _, setAction := range []string{"allow", "deny"} {
var setName = setAction + "_set"
set, err := table.GetSet(setName)
if err != nil {
if nftables.IsNotFound(err) {
var keyType nftables.SetDataType
if tableDef.IsIPv4 {
keyType = nftables.TypeIPAddr
} else if tableDef.IsIPv6 {
keyType = nftables.TypeIP6Addr
}
set, err = table.AddSet(setName, &nftables.SetOptions{
KeyType: keyType,
HasTimeout: true,
})
if err != nil {
return errors.New("create set '" + setName + "' failed: " + err.Error())
}
} else {
return errors.New("get set '" + setName + "' failed: " + err.Error())
}
}
if set == nil {
return errors.New("can not create set '" + setName + "'")
}
if tableDef.IsIPv4 {
if setAction == "allow" {
this.allowIPv4Set = set
} else {
this.denyIPv4Set = set
}
} else if tableDef.IsIPv6 {
if setAction == "allow" {
this.allowIPv6Set = set
} else {
this.denyIPv6Set = set
}
}
// rule
var ruleName = []byte(setAction)
rule, err := chain.GetRuleWithUserData(ruleName)
if err != nil {
if nftables.IsNotFound(err) {
if tableDef.IsIPv4 {
if setAction == "allow" {
rule, err = chain.AddAcceptIPv4SetRule(setName, ruleName)
} else {
rule, err = chain.AddDropIPv4SetRule(setName, ruleName)
}
} else if tableDef.IsIPv6 {
if setAction == "allow" {
rule, err = chain.AddAcceptIPv6SetRule(setName, ruleName)
} else {
rule, err = chain.AddDropIPv6SetRule(setName, ruleName)
}
}
if err != nil {
return errors.New("add rule failed: " + err.Error())
}
} else {
return errors.New("get rule failed: " + err.Error())
}
}
if rule == nil {
return errors.New("can not create rule '" + string(ruleName) + "'")
}
}
}
this.isReady = true
nftablesIsReady = true
nftablesInstance = this
// load firewalld
var firewalld = NewFirewalld()
if firewalld.IsReady() {
this.firewalld = firewalld
}
return nil
}
// Name 名称
func (this *NFTablesFirewall) Name() string {
return "nftables"
}
// IsReady 是否已准备被调用
func (this *NFTablesFirewall) IsReady() bool {
return this.isReady
}
// IsMock 是否为模拟
func (this *NFTablesFirewall) IsMock() bool {
return false
}
// AllowPort 允许端口
func (this *NFTablesFirewall) AllowPort(port int, protocol string) error {
if this.firewalld != nil {
return this.firewalld.AllowPort(port, protocol)
}
return nil
}
// RemovePort 删除端口
func (this *NFTablesFirewall) RemovePort(port int, protocol string) error {
if this.firewalld != nil {
return this.firewalld.RemovePort(port, protocol)
}
return nil
}
// AllowSourceIP Allow把IP加入白名单
func (this *NFTablesFirewall) AllowSourceIP(ip string) error {
var data = net.ParseIP(ip)
if data == nil {
return errors.New("invalid ip '" + ip + "'")
}
if strings.Contains(ip, ":") { // ipv6
if this.allowIPv6Set == nil {
return errors.New("ipv6 ip set is nil")
}
return this.allowIPv6Set.AddElement(data.To16(), nil)
}
// ipv4
if this.allowIPv4Set == nil {
return errors.New("ipv4 ip set is nil")
}
return this.allowIPv4Set.AddElement(data.To4(), nil)
}
// RejectSourceIP 拒绝某个源IP连接
// we did not create set for drop ip, so we reuse DropSourceIP() method here
func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
return this.DropSourceIP(ip, timeoutSeconds)
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
var data = net.ParseIP(ip)
if data == nil {
return errors.New("invalid ip '" + ip + "'")
}
if strings.Contains(ip, ":") { // ipv6
if this.denyIPv6Set == nil {
return errors.New("ipv6 ip set is nil")
}
return this.denyIPv6Set.AddElement(data.To16(), &nftables.ElementOptions{
Timeout: time.Duration(timeoutSeconds) * time.Second,
})
}
// ipv4
if this.denyIPv4Set == nil {
return errors.New("ipv4 ip set is nil")
}
return this.denyIPv4Set.AddElement(data.To4(), &nftables.ElementOptions{
Timeout: time.Duration(timeoutSeconds) * time.Second,
})
}
// RemoveSourceIP 删除某个源IP
func (this *NFTablesFirewall) RemoveSourceIP(ip string) error {
var data = net.ParseIP(ip)
if data == nil {
return errors.New("invalid ip '" + ip + "'")
}
if strings.Contains(ip, ":") { // ipv6
if this.denyIPv6Set != nil {
err := this.denyIPv6Set.DeleteElement(data.To16())
if err != nil {
return err
}
}
if this.allowIPv6Set != nil {
err := this.allowIPv6Set.DeleteElement(data.To16())
if err != nil {
return err
}
}
return nil
}
// ipv4
if this.allowIPv4Set != nil {
err := this.denyIPv4Set.DeleteElement(data.To4())
if err != nil {
return err
}
err = this.allowIPv4Set.DeleteElement(data.To4())
if err != nil {
return err
}
}
return nil
}
// 读取版本号
func (this *NFTablesFirewall) readVersion(nftPath string) string {
var cmd = exec.Command(nftPath, "--version")
var output = &bytes.Buffer{}
cmd.Stdout = output
err := cmd.Run()
if err != nil {
return ""
}
var outputString = output.String()
var versionMatches = regexp.MustCompile(`nftables v([\d.]+)`).FindStringSubmatch(outputString)
if len(versionMatches) <= 1 {
return ""
}
return versionMatches[1]
}

View File

@@ -0,0 +1,61 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build !linux
// +build !linux
package firewalls
import (
"errors"
)
func NewNFTablesFirewall() (*NFTablesFirewall, error) {
return nil, errors.New("not implemented")
}
type NFTablesFirewall struct {
}
// Name 名称
func (this *NFTablesFirewall) Name() string {
return "nftables"
}
// IsReady 是否已准备被调用
func (this *NFTablesFirewall) IsReady() bool {
return false
}
// IsMock 是否为模拟
func (this *NFTablesFirewall) IsMock() bool {
return true
}
// AllowPort 允许端口
func (this *NFTablesFirewall) AllowPort(port int, protocol string) error {
return nil
}
// RemovePort 删除端口
func (this *NFTablesFirewall) RemovePort(port int, protocol string) error {
return nil
}
// AllowSourceIP Allow把IP加入白名单
func (this *NFTablesFirewall) AllowSourceIP(ip string) error {
return nil
}
// RejectSourceIP 拒绝某个源IP连接
func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
return nil
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
return nil
}
// RemoveSourceIP 删除某个源IP
func (this *NFTablesFirewall) RemoveSourceIP(ip string) error {
return nil
}

View File

@@ -0,0 +1 @@
build_remote.sh

View File

@@ -0,0 +1,370 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables
import (
"bytes"
"errors"
nft "github.com/google/nftables"
"github.com/google/nftables/expr"
)
const MaxChainNameLength = 31
type RuleOptions struct {
Exprs []expr.Any
UserData []byte
}
// Chain chain object in table
type Chain struct {
conn *Conn
rawTable *nft.Table
rawChain *nft.Chain
}
func NewChain(conn *Conn, rawTable *nft.Table, rawChain *nft.Chain) *Chain {
return &Chain{
conn: conn,
rawTable: rawTable,
rawChain: rawChain,
}
}
func (this *Chain) Raw() *nft.Chain {
return this.rawChain
}
func (this *Chain) Name() string {
return this.rawChain.Name
}
func (this *Chain) AddRule(options *RuleOptions) (*Rule, error) {
var rawRule = this.conn.Raw().AddRule(&nft.Rule{
Table: this.rawTable,
Chain: this.rawChain,
Exprs: options.Exprs,
UserData: options.UserData,
})
err := this.conn.Commit()
if err != nil {
return nil, err
}
return NewRule(rawRule), nil
}
func (this *Chain) AddAcceptIPv4Rule(ip []byte, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 12,
Len: 4,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ip,
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
UserData: userData,
})
}
func (this *Chain) AddAcceptIPv6Rule(ip []byte, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 8,
Len: 16,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ip,
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
UserData: userData,
})
}
func (this *Chain) AddDropIPv4Rule(ip []byte, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 12,
Len: 4,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ip,
},
&expr.Verdict{
Kind: expr.VerdictDrop,
},
},
UserData: userData,
})
}
func (this *Chain) AddDropIPv6Rule(ip []byte, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 8,
Len: 16,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ip,
},
&expr.Verdict{
Kind: expr.VerdictDrop,
},
},
UserData: userData,
})
}
func (this *Chain) AddRejectIPv4Rule(ip []byte, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 12,
Len: 4,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ip,
},
&expr.Reject{},
},
UserData: userData,
})
}
func (this *Chain) AddRejectIPv6Rule(ip []byte, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 8,
Len: 16,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ip,
},
&expr.Reject{},
},
UserData: userData,
})
}
func (this *Chain) AddAcceptIPv4SetRule(setName string, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 12,
Len: 4,
},
&expr.Lookup{
SourceRegister: 1,
SetName: setName,
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
UserData: userData,
})
}
func (this *Chain) AddAcceptIPv6SetRule(setName string, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 8,
Len: 16,
},
&expr.Lookup{
SourceRegister: 1,
SetName: setName,
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
UserData: userData,
})
}
func (this *Chain) AddDropIPv4SetRule(setName string, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 12,
Len: 4,
},
&expr.Lookup{
SourceRegister: 1,
SetName: setName,
},
&expr.Verdict{
Kind: expr.VerdictDrop,
},
},
UserData: userData,
})
}
func (this *Chain) AddDropIPv6SetRule(setName string, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 8,
Len: 16,
},
&expr.Lookup{
SourceRegister: 1,
SetName: setName,
},
&expr.Verdict{
Kind: expr.VerdictDrop,
},
},
UserData: userData,
})
}
func (this *Chain) AddRejectIPv4SetRule(setName string, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 12,
Len: 4,
},
&expr.Lookup{
SourceRegister: 1,
SetName: setName,
},
&expr.Reject{},
},
UserData: userData,
})
}
func (this *Chain) AddRejectIPv6SetRule(setName string, userData []byte) (*Rule, error) {
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
Offset: 8,
Len: 16,
},
&expr.Lookup{
SourceRegister: 1,
SetName: setName,
},
&expr.Reject{},
},
UserData: userData,
})
}
func (this *Chain) AddAcceptInterfaceRule(interfaceName string, userData []byte) (*Rule, error) {
if len(interfaceName) >= 16 {
return nil, errors.New("invalid interface name '" + interfaceName + "'")
}
var ifname = make([]byte, 16)
copy(ifname, interfaceName+"\x00")
return this.AddRule(&RuleOptions{
Exprs: []expr.Any{
&expr.Meta{
Key: expr.MetaKeyIIFNAME,
Register: 1,
},
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: ifname,
},
&expr.Verdict{
Kind: expr.VerdictAccept,
},
},
UserData: userData,
})
}
func (this *Chain) GetRuleWithUserData(userData []byte) (*Rule, error) {
rawRules, err := this.conn.Raw().GetRule(this.rawTable, this.rawChain)
if err != nil {
return nil, err
}
for _, rawRule := range rawRules {
if bytes.Compare(rawRule.UserData, userData) == 0 {
return NewRule(rawRule), nil
}
}
return nil, ErrRuleNotFound
}
func (this *Chain) GetRules() ([]*Rule, error) {
rawRules, err := this.conn.Raw().GetRule(this.rawTable, this.rawChain)
if err != nil {
return nil, err
}
var result = []*Rule{}
for _, rawRule := range rawRules {
result = append(result, NewRule(rawRule))
}
return result, nil
}
func (this *Chain) DeleteRule(rule *Rule) error {
err := this.conn.Raw().DelRule(rule.Raw())
if err != nil {
return err
}
return this.conn.Commit()
}
func (this *Chain) Flush() error {
this.conn.Raw().FlushChain(this.rawChain)
return this.conn.Commit()
}

View File

@@ -0,0 +1,13 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nftables
import nft "github.com/google/nftables"
type ChainPolicy = nft.ChainPolicy
// Possible ChainPolicy values.
const (
ChainPolicyDrop = nft.ChainPolicyDrop
ChainPolicyAccept = nft.ChainPolicyAccept
)

View File

@@ -0,0 +1,130 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables_test
import (
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"net"
"testing"
)
func getIPv4Chain(t *testing.T) *nftables.Chain {
var conn = nftables.NewConn()
table, err := conn.GetTable("test_ipv4", nftables.TableFamilyIPv4)
if err != nil {
if err == nftables.ErrTableNotFound {
table, err = conn.AddIPv4Table("test_ipv4")
if err != nil {
t.Fatal(err)
}
} else {
t.Fatal(err)
}
}
chain, err := table.GetChain("test_chain")
if err != nil {
if err == nftables.ErrChainNotFound {
chain, err = table.AddAcceptChain("test_chain")
}
}
if err != nil {
t.Fatal(err)
}
return chain
}
func TestChain_AddAcceptIPRule(t *testing.T) {
var chain = getIPv4Chain(t)
_, err := chain.AddAcceptIPv4Rule(net.ParseIP("192.168.2.40").To4(), nil)
if err != nil {
t.Fatal(err)
}
}
func TestChain_AddDropIPRule(t *testing.T) {
var chain = getIPv4Chain(t)
_, err := chain.AddDropIPv4Rule(net.ParseIP("192.168.2.31").To4(), nil)
if err != nil {
t.Fatal(err)
}
}
func TestChain_AddAcceptSetRule(t *testing.T) {
var chain = getIPv4Chain(t)
_, err := chain.AddAcceptIPv4SetRule("ipv4_black_set", nil)
if err != nil {
t.Fatal(err)
}
}
func TestChain_AddDropSetRule(t *testing.T) {
var chain = getIPv4Chain(t)
_, err := chain.AddDropIPv4SetRule("ipv4_black_set", nil)
if err != nil {
t.Fatal(err)
}
}
func TestChain_AddRejectSetRule(t *testing.T) {
var chain = getIPv4Chain(t)
_, err := chain.AddRejectIPv4SetRule("ipv4_black_set", nil)
if err != nil {
t.Fatal(err)
}
}
func TestChain_GetRuleWithUserData(t *testing.T) {
var chain = getIPv4Chain(t)
rule, err := chain.GetRuleWithUserData([]byte("test"))
if err != nil {
if err == nftables.ErrRuleNotFound {
t.Log("rule not found")
return
} else {
t.Fatal(err)
}
}
t.Log("rule:", rule)
}
func TestChain_GetRules(t *testing.T) {
var chain = getIPv4Chain(t)
rules, err := chain.GetRules()
if err != nil {
t.Fatal(err)
}
for _, rule := range rules {
t.Log("handle:", rule.Handle(), "set name:", rule.LookupSetName(),
"verdict:", rule.VerDict(), "user data:", string(rule.UserData()))
}
}
func TestChain_DeleteRule(t *testing.T) {
var chain = getIPv4Chain(t)
rule, err := chain.GetRuleWithUserData([]byte("test"))
if err != nil {
if err == nftables.ErrRuleNotFound {
t.Log("rule not found")
return
}
t.Fatal(err)
}
err = chain.DeleteRule(rule)
if err != nil {
t.Fatal(err)
}
}
func TestChain_Flush(t *testing.T) {
var chain = getIPv4Chain(t)
err := chain.Flush()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -0,0 +1,84 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables
import (
"errors"
nft "github.com/google/nftables"
"github.com/iwind/TeaGo/types"
)
const MaxTableNameLength = 27
type Conn struct {
rawConn *nft.Conn
}
func NewConn() *Conn {
return &Conn{
rawConn: &nft.Conn{},
}
}
func (this *Conn) Raw() *nft.Conn {
return this.rawConn
}
func (this *Conn) GetTable(name string, family TableFamily) (*Table, error) {
rawTables, err := this.rawConn.ListTables()
if err != nil {
return nil, err
}
for _, rawTable := range rawTables {
if rawTable.Name == name && rawTable.Family == family {
return NewTable(this, rawTable), nil
}
}
return nil, ErrTableNotFound
}
func (this *Conn) AddTable(name string, family TableFamily) (*Table, error) {
if len(name) > MaxTableNameLength {
return nil, errors.New("table name too long (max " + types.String(MaxTableNameLength) + ")")
}
var rawTable = this.rawConn.AddTable(&nft.Table{
Family: family,
Name: name,
})
err := this.Commit()
if err != nil {
return nil, err
}
return NewTable(this, rawTable), nil
}
func (this *Conn) AddIPv4Table(name string) (*Table, error) {
return this.AddTable(name, TableFamilyIPv4)
}
func (this *Conn) AddIPv6Table(name string) (*Table, error) {
return this.AddTable(name, TableFamilyIPv6)
}
func (this *Conn) DeleteTable(name string, family TableFamily) error {
table, err := this.GetTable(name, family)
if err != nil {
if err == ErrTableNotFound {
return nil
}
return err
}
this.rawConn.DelTable(table.Raw())
return this.Commit()
}
func (this *Conn) Commit() error {
return this.rawConn.Flush()
}

View File

@@ -0,0 +1,78 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables_test
import (
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"os/exec"
"testing"
)
func TestConn_Test(t *testing.T) {
_, err := exec.LookPath("nft")
if err == nil {
t.Log("ok")
return
}
t.Log(err)
}
func TestConn_GetTable_NotFound(t *testing.T) {
var conn = nftables.NewConn()
table, err := conn.GetTable("a", nftables.TableFamilyIPv4)
if err != nil {
if err == nftables.ErrTableNotFound {
t.Log("table not found")
} else {
t.Fatal(err)
}
} else {
t.Log("table:", table)
}
}
func TestConn_GetTable(t *testing.T) {
var conn = nftables.NewConn()
table, err := conn.GetTable("myFilter", nftables.TableFamilyIPv4)
if err != nil {
if err == nftables.ErrTableNotFound {
t.Log("table not found")
} else {
t.Fatal(err)
}
} else {
t.Log("table:", table)
}
}
func TestConn_AddTable(t *testing.T) {
var conn = nftables.NewConn()
{
table, err := conn.AddIPv4Table("test_ipv4")
if err != nil {
t.Fatal(err)
}
t.Log(table.Name())
}
{
table, err := conn.AddIPv6Table("test_ipv6")
if err != nil {
t.Fatal(err)
}
t.Log(table.Name())
}
}
func TestConn_DeleteTable(t *testing.T) {
var conn = nftables.NewConn()
err := conn.DeleteTable("test_ipv4", nftables.TableFamilyIPv4)
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -0,0 +1,8 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables
type Element struct {
}

View File

@@ -0,0 +1,19 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables
import "errors"
var ErrTableNotFound = errors.New("table not found")
var ErrChainNotFound = errors.New("chain not found")
var ErrSetNotFound = errors.New("set not found")
var ErrRuleNotFound = errors.New("rule not found")
func IsNotFound(err error) bool {
if err == nil {
return false
}
return err == ErrTableNotFound || err == ErrChainNotFound || err == ErrSetNotFound || err == ErrRuleNotFound
}

View File

@@ -0,0 +1,18 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nftables
import (
nft "github.com/google/nftables"
)
type TableFamily = nft.TableFamily
const (
TableFamilyINet TableFamily = nft.TableFamilyINet
TableFamilyIPv4 TableFamily = nft.TableFamilyIPv4
TableFamilyIPv6 TableFamily = nft.TableFamilyIPv6
TableFamilyARP TableFamily = nft.TableFamilyARP
TableFamilyNetdev TableFamily = nft.TableFamilyNetdev
TableFamilyBridge TableFamily = nft.TableFamilyBridge
)

View File

@@ -0,0 +1,51 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nftables
import (
nft "github.com/google/nftables"
"github.com/google/nftables/expr"
)
type Rule struct {
rawRule *nft.Rule
}
func NewRule(rawRule *nft.Rule) *Rule {
return &Rule{
rawRule: rawRule,
}
}
func (this *Rule) Raw() *nft.Rule {
return this.rawRule
}
func (this *Rule) LookupSetName() string {
for _, e := range this.rawRule.Exprs {
exp, ok := e.(*expr.Lookup)
if ok {
return exp.SetName
}
}
return ""
}
func (this *Rule) VerDict() expr.VerdictKind {
for _, e := range this.rawRule.Exprs {
exp, ok := e.(*expr.Verdict)
if ok {
return exp.Kind
}
}
return -100
}
func (this *Rule) Handle() uint64 {
return this.rawRule.Handle
}
func (this *Rule) UserData() []byte {
return this.rawRule.UserData
}

View File

@@ -0,0 +1,161 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/utils"
nft "github.com/google/nftables"
"net"
"strings"
"time"
)
const MaxSetNameLength = 15
type SetOptions struct {
Id uint32
HasTimeout bool
Timeout time.Duration
KeyType SetDataType
DataType SetDataType
Constant bool
Interval bool
Anonymous bool
IsMap bool
}
type ElementOptions struct {
Timeout time.Duration
}
type Set struct {
conn *Conn
rawSet *nft.Set
batch *SetBatch
}
func NewSet(conn *Conn, rawSet *nft.Set) *Set {
return &Set{
conn: conn,
rawSet: rawSet,
batch: &SetBatch{
conn: conn,
rawSet: rawSet,
},
}
}
func (this *Set) Raw() *nft.Set {
return this.rawSet
}
func (this *Set) Name() string {
return this.rawSet.Name
}
func (this *Set) AddElement(key []byte, options *ElementOptions) error {
var rawElement = nft.SetElement{
Key: key,
}
if options != nil {
rawElement.Timeout = options.Timeout
}
err := this.conn.Raw().SetAddElements(this.rawSet, []nft.SetElement{
rawElement,
})
if err != nil {
return err
}
err = this.conn.Commit()
if err != nil {
// retry if exists
if strings.Contains(err.Error(), "file exists") {
deleteErr := this.conn.Raw().SetDeleteElements(this.rawSet, []nft.SetElement{
{
Key: key,
},
})
if deleteErr == nil {
err = this.conn.Raw().SetAddElements(this.rawSet, []nft.SetElement{
rawElement,
})
if err == nil {
err = this.conn.Commit()
}
}
}
}
return err
}
func (this *Set) AddIPElement(ip string, options *ElementOptions) error {
var ipObj = net.ParseIP(ip)
if ipObj == nil {
return errors.New("invalid ip '" + ip + "'")
}
if utils.IsIPv4(ip) {
return this.AddElement(ipObj.To4(), options)
} else {
return this.AddElement(ipObj.To16(), options)
}
}
func (this *Set) DeleteElement(key []byte) error {
err := this.conn.Raw().SetDeleteElements(this.rawSet, []nft.SetElement{
{
Key: key,
},
})
if err != nil {
return err
}
err = this.conn.Commit()
if err != nil {
if strings.Contains(err.Error(), "no such file or directory") {
err = nil
}
}
return err
}
func (this *Set) DeleteIPElement(ip string) error {
var ipObj = net.ParseIP(ip)
if ipObj == nil {
return errors.New("invalid ip '" + ip + "'")
}
if utils.IsIPv4(ip) {
return this.DeleteElement(ipObj.To4())
} else {
return this.DeleteElement(ipObj.To16())
}
}
func (this *Set) Batch() *SetBatch {
return this.batch
}
func (this *Set) GetIPElements() ([]string, error) {
elements, err := this.conn.Raw().GetSetElements(this.rawSet)
if err != nil {
return nil, err
}
var result = []string{}
for _, element := range elements {
result = append(result, net.IP(element.Key).String())
}
return result, nil
}
// not work current time
/**func (this *Set) Flush() error {
this.conn.Raw().FlushSet(this.rawSet)
return this.conn.Commit()
}**/

View File

@@ -0,0 +1,36 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nftables
import (
nft "github.com/google/nftables"
)
type SetBatch struct {
conn *Conn
rawSet *nft.Set
}
func (this *SetBatch) AddElement(key []byte, options *ElementOptions) error {
var rawElement = nft.SetElement{
Key: key,
}
if options != nil {
rawElement.Timeout = options.Timeout
}
return this.conn.Raw().SetAddElements(this.rawSet, []nft.SetElement{
rawElement,
})
}
func (this *SetBatch) DeleteElement(key []byte) error {
return this.conn.Raw().SetDeleteElements(this.rawSet, []nft.SetElement{
{
Key: key,
},
})
}
func (this *SetBatch) Commit() error {
return this.conn.Commit()
}

View File

@@ -0,0 +1,57 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nftables
import nft "github.com/google/nftables"
type SetDataType = nft.SetDatatype
var (
TypeInvalid = nft.TypeInvalid
TypeVerdict = nft.TypeVerdict
TypeNFProto = nft.TypeNFProto
TypeBitmask = nft.TypeBitmask
TypeInteger = nft.TypeInteger
TypeString = nft.TypeString
TypeLLAddr = nft.TypeLLAddr
TypeIPAddr = nft.TypeIPAddr
TypeIP6Addr = nft.TypeIP6Addr
TypeEtherAddr = nft.TypeEtherAddr
TypeEtherType = nft.TypeEtherType
TypeARPOp = nft.TypeARPOp
TypeInetProto = nft.TypeInetProto
TypeInetService = nft.TypeInetService
TypeICMPType = nft.TypeICMPType
TypeTCPFlag = nft.TypeTCPFlag
TypeDCCPPktType = nft.TypeDCCPPktType
TypeMHType = nft.TypeMHType
TypeTime = nft.TypeTime
TypeMark = nft.TypeMark
TypeIFIndex = nft.TypeIFIndex
TypeARPHRD = nft.TypeARPHRD
TypeRealm = nft.TypeRealm
TypeClassID = nft.TypeClassID
TypeUID = nft.TypeUID
TypeGID = nft.TypeGID
TypeCTState = nft.TypeCTState
TypeCTDir = nft.TypeCTDir
TypeCTStatus = nft.TypeCTStatus
TypeICMP6Type = nft.TypeICMP6Type
TypeCTLabel = nft.TypeCTLabel
TypePktType = nft.TypePktType
TypeICMPCode = nft.TypeICMPCode
TypeICMPV6Code = nft.TypeICMPV6Code
TypeICMPXCode = nft.TypeICMPXCode
TypeDevGroup = nft.TypeDevGroup
TypeDSCP = nft.TypeDSCP
TypeECN = nft.TypeECN
TypeFIBAddr = nft.TypeFIBAddr
TypeBoolean = nft.TypeBoolean
TypeCTEventBit = nft.TypeCTEventBit
TypeIFName = nft.TypeIFName
TypeIGMPType = nft.TypeIGMPType
TypeTimeDate = nft.TypeTimeDate
TypeTimeHour = nft.TypeTimeHour
TypeTimeDay = nft.TypeTimeDay
TypeCGroupV2 = nft.TypeCGroupV2
)

View File

@@ -0,0 +1,110 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nftables_test
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/iwind/TeaGo/types"
"github.com/mdlayher/netlink"
"net"
"testing"
"time"
)
func getIPv4Set(t *testing.T) *nftables.Set {
var table = getIPv4Table(t)
set, err := table.GetSet("test_ipv4_set")
if err != nil {
if err == nftables.ErrSetNotFound {
set, err = table.AddSet("test_ipv4_set", &nftables.SetOptions{
KeyType: nftables.TypeIPAddr,
HasTimeout: true,
})
if err != nil {
t.Fatal(err)
}
} else {
t.Fatal(err)
}
}
return set
}
func TestSet_AddElement(t *testing.T) {
var set = getIPv4Set(t)
err := set.AddElement(net.ParseIP("192.168.2.31").To4(), &nftables.ElementOptions{Timeout: 86400 * time.Second})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestSet_DeleteElement(t *testing.T) {
var set = getIPv4Set(t)
err := set.DeleteElement(net.ParseIP("192.168.2.31").To4())
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestSet_Batch(t *testing.T) {
var batch = getIPv4Set(t).Batch()
for _, ip := range []string{"192.168.2.30", "192.168.2.31", "192.168.2.32", "192.168.2.33", "192.168.2.34"} {
var ipData = net.ParseIP(ip).To4()
//err := batch.DeleteElement(ipData)
//if err != nil {
// t.Fatal(err)
//}
err := batch.AddElement(ipData, &nftables.ElementOptions{Timeout: 10 * time.Second})
if err != nil {
t.Fatal(err)
}
}
err := batch.Commit()
if err != nil {
t.Logf("%#v", errors.Unwrap(err).(*netlink.OpError))
t.Fatal(err)
}
t.Log("ok")
}
func TestSet_Add_Many(t *testing.T) {
var set = getIPv4Set(t)
for i := 0; i < 255; i++ {
t.Log(i)
for j := 0; j < 255; j++ {
var ip = "192.167." + types.String(i) + "." + types.String(j)
var ipData = net.ParseIP(ip).To4()
err := set.Batch().AddElement(ipData, &nftables.ElementOptions{Timeout: 3600 * time.Second})
if err != nil {
t.Fatal(err)
}
if j%10 == 0 {
err = set.Batch().Commit()
if err != nil {
t.Fatal(err)
}
}
}
err := set.Batch().Commit()
if err != nil {
t.Fatal(err)
}
}
t.Log("ok")
}
/**func TestSet_Flush(t *testing.T) {
var set = getIPv4Set(t)
err := set.Flush()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}**/

View File

@@ -0,0 +1,157 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables
import (
"errors"
nft "github.com/google/nftables"
"github.com/iwind/TeaGo/types"
"strings"
)
type Table struct {
conn *Conn
rawTable *nft.Table
}
func NewTable(conn *Conn, rawTable *nft.Table) *Table {
return &Table{
conn: conn,
rawTable: rawTable,
}
}
func (this *Table) Raw() *nft.Table {
return this.rawTable
}
func (this *Table) Name() string {
return this.rawTable.Name
}
func (this *Table) Family() TableFamily {
return this.rawTable.Family
}
func (this *Table) GetChain(name string) (*Chain, error) {
rawChains, err := this.conn.Raw().ListChains()
if err != nil {
return nil, err
}
for _, rawChain := range rawChains {
// must compare table name
if rawChain.Name == name && rawChain.Table.Name == this.rawTable.Name {
return NewChain(this.conn, this.rawTable, rawChain), nil
}
}
return nil, ErrChainNotFound
}
func (this *Table) AddChain(name string, chainPolicy *ChainPolicy) (*Chain, error) {
if len(name) > MaxChainNameLength {
return nil, errors.New("chain name too long (max " + types.String(MaxChainNameLength) + ")")
}
var rawChain = this.conn.Raw().AddChain(&nft.Chain{
Name: name,
Table: this.rawTable,
Hooknum: nft.ChainHookInput,
Priority: nft.ChainPriorityFilter,
Type: nft.ChainTypeFilter,
Policy: chainPolicy,
})
err := this.conn.Commit()
if err != nil {
return nil, err
}
return NewChain(this.conn, this.rawTable, rawChain), nil
}
func (this *Table) AddAcceptChain(name string) (*Chain, error) {
var policy = ChainPolicyAccept
return this.AddChain(name, &policy)
}
func (this *Table) AddDropChain(name string) (*Chain, error) {
var policy = ChainPolicyDrop
return this.AddChain(name, &policy)
}
func (this *Table) DeleteChain(name string) error {
chain, err := this.GetChain(name)
if err != nil {
if err == ErrChainNotFound {
return nil
}
return err
}
this.conn.Raw().DelChain(chain.Raw())
return this.conn.Commit()
}
func (this *Table) GetSet(name string) (*Set, error) {
rawSet, err := this.conn.Raw().GetSetByName(this.rawTable, name)
if err != nil {
if strings.Contains(err.Error(), "no such file or directory") {
return nil, ErrSetNotFound
}
return nil, err
}
return NewSet(this.conn, rawSet), nil
}
func (this *Table) AddSet(name string, options *SetOptions) (*Set, error) {
if len(name) > MaxSetNameLength {
return nil, errors.New("set name too long (max " + types.String(MaxSetNameLength) + ")")
}
if options == nil {
options = &SetOptions{}
}
var rawSet = &nft.Set{
Table: this.rawTable,
ID: options.Id,
Name: name,
Anonymous: options.Anonymous,
Constant: options.Constant,
Interval: options.Interval,
IsMap: options.IsMap,
HasTimeout: options.HasTimeout,
Timeout: options.Timeout,
KeyType: options.KeyType,
DataType: options.DataType,
}
err := this.conn.Raw().AddSet(rawSet, nil)
if err != nil {
return nil, err
}
err = this.conn.Commit()
if err != nil {
return nil, err
}
return NewSet(this.conn, rawSet), nil
}
func (this *Table) DeleteSet(name string) error {
set, err := this.GetSet(name)
if err != nil {
if err == ErrSetNotFound {
return nil
}
return err
}
this.conn.Raw().DelSet(set.Raw())
return this.conn.Commit()
}
func (this *Table) Flush() error {
this.conn.Raw().FlushTable(this.rawTable)
return this.conn.Commit()
}

View File

@@ -0,0 +1,140 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables_test
import (
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"testing"
)
func getIPv4Table(t *testing.T) *nftables.Table {
var conn = nftables.NewConn()
table, err := conn.GetTable("test_ipv4", nftables.TableFamilyIPv4)
if err != nil {
if err == nftables.ErrTableNotFound {
table, err = conn.AddIPv4Table("test_ipv4")
if err != nil {
t.Fatal(err)
}
} else {
t.Fatal(err)
}
}
return table
}
func TestTable_AddChain(t *testing.T) {
var table = getIPv4Table(t)
{
chain, err := table.AddChain("test_default_chain", nil)
if err != nil {
t.Fatal(err)
}
t.Log("created:", chain.Name())
}
{
chain, err := table.AddAcceptChain("test_accept_chain")
if err != nil {
t.Fatal(err)
}
t.Log("created:", chain.Name())
}
// Do not test drop chain before adding accept rule, you will drop yourself!!!!!!!
/**{
chain, err := table.AddDropChain("test_drop_chain")
if err != nil {
t.Fatal(err)
}
t.Log("created:", chain.Name())
}**/
}
func TestTable_GetChain(t *testing.T) {
var table = getIPv4Table(t)
for _, chainName := range []string{"not_found_chain", "test_default_chain"} {
chain, err := table.GetChain(chainName)
if err != nil {
if err == nftables.ErrChainNotFound {
t.Log(chainName, ":", "not found")
} else {
t.Fatal(err)
}
} else {
t.Log(chainName, ":", chain)
}
}
}
func TestTable_DeleteChain(t *testing.T) {
var table = getIPv4Table(t)
err := table.DeleteChain("test_default_chain")
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestTable_AddSet(t *testing.T) {
var table = getIPv4Table(t)
{
set, err := table.AddSet("ipv4_black_set", &nftables.SetOptions{
HasTimeout: false,
KeyType: nftables.TypeIPAddr,
})
if err != nil {
t.Fatal(err)
}
t.Log(set.Name())
}
{
set, err := table.AddSet("ipv6_black_set", &nftables.SetOptions{
HasTimeout: true,
//Timeout: 3600 * time.Second,
KeyType: nftables.TypeIP6Addr,
})
if err != nil {
t.Fatal(err)
}
t.Log(set.Name())
}
}
func TestTable_GetSet(t *testing.T) {
var table = getIPv4Table(t)
for _, setName := range []string{"not_found_set", "ipv4_black_set"} {
set, err := table.GetSet(setName)
if err != nil {
if err == nftables.ErrSetNotFound {
t.Log(setName, ": not found")
} else {
t.Fatal(err)
}
} else {
t.Log(setName, ":", set)
}
}
}
func TestTable_DeleteSet(t *testing.T) {
var table = getIPv4Table(t)
err := table.DeleteSet("ipv4_black_set")
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func TestTable_Flush(t *testing.T) {
var table = getIPv4Table(t)
err := table.Flush()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -48,7 +48,7 @@ func (this *IPListDB) init() error {
if err != nil {
return err
}
remotelogs.Println("CACHE", "create cache dir '"+this.dir+"'")
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")

View File

@@ -3,12 +3,27 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
)
// AllowIP 检查IP是否被允许访问
// 如果一个IP不在任何名单中则允许访问
func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool) {
if !Tea.IsTesting() { // 如果在测试环境,我们不加入一些白名单,以便于可以在本地和局域网正常测试
// 放行lo
if ip == "127.0.0.1" || ip == "::1" {
return true, true
}
// check node
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err == nil && nodeConfig.IPIsAutoAllowed(ip) {
return true, true
}
}
var ipLong = utils.IP2Long(ip)
if ipLong == 0 {
return false, false

View File

@@ -23,7 +23,7 @@ import (
"time"
)
const MaxQueueSize = 10240
const MaxQueueSize = 2048
// Task 单个指标任务
// 数据库存储:
@@ -58,7 +58,7 @@ type Task struct {
timeMap map[string]zero.Zero // time => bool
serverIdMapLocker sync.Mutex
statsMap map[string]*Stat
statsMap map[string]*Stat // 待写入队列hash => *Stat
statsLocker sync.Mutex
statsTicker *utils.Ticker
}
@@ -210,7 +210,7 @@ func (this *Task) Start() error {
var tr = trackers.Begin("[METRIC]UPLOAD_STATS")
err := this.Upload(1 * time.Second)
tr.End()
if err != nil {
if err != nil && !rpc.IsConnError(err) {
remotelogs.Error("METRIC", "upload stats failed: "+err.Error())
}
}

View File

@@ -1,32 +1,30 @@
package nodes
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/configs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
"io"
"net"
"net/http"
"github.com/iwind/TeaGo/maps"
"net/url"
"os/exec"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
@@ -111,14 +109,12 @@ func (this *APIStream) loop() error {
err = this.handleStatCache(message)
case messageconfigs.MessageCodeCleanCache: // 清理缓存
err = this.handleCleanCache(message)
case messageconfigs.MessageCodePurgeCache: // 删除缓存
err = this.handlePurgeCache(message)
case messageconfigs.MessageCodePreheatCache: // 预热缓存
err = this.handlePreheatCache(message)
case messageconfigs.MessageCodeNewNodeTask: // 有新的任务
err = this.handleNewNodeTask(message)
case messageconfigs.MessageCodeCheckSystemdService: // 检查Systemd服务
err = this.handleCheckSystemdService(message)
case messageconfigs.MessageCodeCheckLocalFirewall: // 检查本地防火墙
err = this.handleCheckLocalFirewall(message)
case messageconfigs.MessageCodeChangeAPINode: // 修改API节点地址
err = this.handleChangeAPINode(message)
default:
@@ -328,224 +324,6 @@ func (this *APIStream) handleCleanCache(message *pb.NodeStreamMessage) error {
return nil
}
// 删除缓存
func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
msg := &messageconfigs.PurgeCacheMessage{}
err := json.Unmarshal(message.DataJSON, msg)
if err != nil {
this.replyFail(message.RequestId, "decode message data failed: "+err.Error())
return err
}
storage, shouldStop, err := this.cacheStorage(message, msg.CachePolicyJSON)
if err != nil {
return err
}
if shouldStop {
defer func() {
storage.Stop()
}()
}
// WEBP缓存
if msg.Type == "file" {
var keys = msg.Keys
for _, key := range keys {
keys = append(keys,
key+caches.SuffixMethod+"HEAD",
key+caches.SuffixWebP,
key+caches.SuffixPartial,
)
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
keys = append(keys, key+caches.SuffixCompression+encoding)
keys = append(keys, key+caches.SuffixWebP+caches.SuffixCompression+encoding)
}
}
msg.Keys = keys
}
err = storage.Purge(msg.Keys, msg.Type)
if err != nil {
this.replyFail(message.RequestId, "purge keys failed: "+err.Error())
return err
}
this.replyOk(message.RequestId, "ok")
return nil
}
// 预热缓存
func (this *APIStream) handlePreheatCache(message *pb.NodeStreamMessage) error {
msg := &messageconfigs.PreheatCacheMessage{}
err := json.Unmarshal(message.DataJSON, msg)
if err != nil {
this.replyFail(message.RequestId, "decode message data failed: "+err.Error())
return err
}
storage, shouldStop, err := this.cacheStorage(message, msg.CachePolicyJSON)
if err != nil {
return err
}
if shouldStop {
defer func() {
storage.Stop()
}()
}
if len(msg.Keys) == 0 {
this.replyOk(message.RequestId, "ok")
return nil
}
wg := sync.WaitGroup{}
wg.Add(len(msg.Keys))
client := &http.Client{
Timeout: 30 * time.Second, // TODO 可以设置请求超时时间
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
_, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
return net.Dial(network, "127.0.0.1:"+port)
},
MaxIdleConns: 4096,
MaxIdleConnsPerHost: 32,
MaxConnsPerHost: 32,
IdleConnTimeout: 2 * time.Minute,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 0,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
defer client.CloseIdleConnections()
errorMessages := []string{}
locker := sync.Mutex{}
for _, key := range msg.Keys {
go func(key string) {
defer wg.Done()
req, err := http.NewRequest(http.MethodGet, key, nil)
if err != nil {
locker.Lock()
errorMessages = append(errorMessages, "invalid url: "+key+": "+err.Error())
locker.Unlock()
return
}
// TODO 可以在管理界面自定义Header
req.Header.Set("X-Cache-Action", "preheat")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36")
req.Header.Set("Accept-Encoding", "gzip, deflate, br") // TODO 这里需要记录下缓存是否为gzip的
resp, err := client.Do(req)
if err != nil {
locker.Lock()
errorMessages = append(errorMessages, "request failed: "+key+": "+err.Error())
locker.Unlock()
return
}
if resp.StatusCode != 200 {
locker.Lock()
errorMessages = append(errorMessages, "request failed: "+key+": status code '"+strconv.Itoa(resp.StatusCode)+"'")
locker.Unlock()
return
}
defer func() {
_ = resp.Body.Close()
}()
// 检查最大内容长度
// TODO 需要解决Chunked Transfer Encoding的长度判断问题
maxSize := storage.Policy().MaxSizeBytes()
if maxSize > 0 && resp.ContentLength > maxSize {
locker.Lock()
errorMessages = append(errorMessages, "request failed: the content is too larger than policy setting")
locker.Unlock()
return
}
expiredAt := time.Now().Unix() + 8600
writer, err := storage.OpenWriter(key, expiredAt, 200, resp.ContentLength, -1, false) // TODO 可以设置缓存过期时间
if err != nil {
locker.Lock()
errorMessages = append(errorMessages, "open cache writer failed: "+key+": "+err.Error())
locker.Unlock()
return
}
buf := make([]byte, 16*1024)
isClosed := false
// 写入Header
for k, v := range resp.Header {
for _, v1 := range v {
_, err = writer.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
locker.Lock()
errorMessages = append(errorMessages, "write failed: "+key+": "+err.Error())
locker.Unlock()
return
}
}
}
// 写入Body
for {
n, err := resp.Body.Read(buf)
if n > 0 {
_, writerErr := writer.Write(buf[:n])
if writerErr != nil {
locker.Lock()
errorMessages = append(errorMessages, "write failed: "+key+": "+writerErr.Error())
locker.Unlock()
break
}
}
if err != nil {
if err == io.EOF {
err = writer.Close()
if err == nil {
storage.AddToList(&caches.Item{
Type: writer.ItemType(),
Key: key,
ExpiredAt: expiredAt,
})
}
isClosed = true
} else {
locker.Lock()
errorMessages = append(errorMessages, "read url failed: "+key+": "+err.Error())
locker.Unlock()
}
break
}
}
if !isClosed {
_ = writer.Close()
}
}(key)
}
wg.Wait()
if len(errorMessages) > 0 {
this.replyFail(message.RequestId, strings.Join(errorMessages, ", "))
return nil
}
this.replyOk(message.RequestId, "ok")
return nil
}
// 处理配置变化
func (this *APIStream) handleNewNodeTask(message *pb.NodeStreamMessage) error {
select {
@@ -569,7 +347,7 @@ func (this *APIStream) handleCheckSystemdService(message *pb.NodeStreamMessage)
return nil
}
cmd := utils.NewCommandExecutor()
var cmd = utils.NewCommandExecutor()
shortName := teaconst.SystemdServiceName
cmd.Add(systemctl, "is-enabled", shortName)
output, err := cmd.Run()
@@ -585,6 +363,63 @@ func (this *APIStream) handleCheckSystemdService(message *pb.NodeStreamMessage)
return nil
}
// 检查本地防火墙
func (this *APIStream) handleCheckLocalFirewall(message *pb.NodeStreamMessage) error {
var dataMessage = &messageconfigs.CheckLocalFirewallMessage{}
err := json.Unmarshal(message.DataJSON, dataMessage)
if err != nil {
this.replyFail(message.RequestId, "decode message data failed: "+err.Error())
return nil
}
// nft
if dataMessage.Name == "nftables" {
if runtime.GOOS != "linux" {
this.replyFail(message.RequestId, "not Linux system")
return nil
}
nft, err := exec.LookPath("nft")
if err != nil {
this.replyFail(message.RequestId, "'nft' not found: "+err.Error())
return nil
}
var cmd = exec.Command(nft, "--version")
var output = &bytes.Buffer{}
cmd.Stdout = output
err = cmd.Run()
if err != nil {
this.replyFail(message.RequestId, "get version failed: "+err.Error())
return nil
}
var outputString = output.String()
var versionMatches = regexp.MustCompile(`nftables v([\d.]+)`).FindStringSubmatch(outputString)
if len(versionMatches) <= 1 {
this.replyFail(message.RequestId, "can not get nft version")
return nil
}
var version = versionMatches[1]
var result = maps.Map{
"version": version,
}
var protectionConfig = sharedNodeConfig.DDoSProtection
err = firewalls.SharedDDoSProtectionManager.Apply(protectionConfig)
if err != nil {
this.replyFail(message.RequestId, dataMessage.Name+"was installed, but apply DDoS protection config failed: "+err.Error())
} else {
this.replyOk(message.RequestId, string(result.AsJSON()))
}
} else {
this.replyFail(message.RequestId, "invalid firewall name '"+dataMessage.Name+"'")
}
return nil
}
// 修改API地址
func (this *APIStream) handleChangeAPINode(message *pb.NodeStreamMessage) error {
config, err := configs.LoadAPIConfig()
@@ -660,6 +495,11 @@ func (this *APIStream) replyOk(requestId int64, message string) {
_ = this.stream.Send(&pb.NodeStreamMessage{RequestId: requestId, IsOk: true, Message: message})
}
// 回复成功并包含数据
func (this *APIStream) replyOkData(requestId int64, message string, dataJSON []byte) {
_ = this.stream.Send(&pb.NodeStreamMessage{RequestId: requestId, IsOk: true, Message: message, DataJSON: dataJSON})
}
// 获取缓存存取对象
func (this *APIStream) cacheStorage(message *pb.NodeStreamMessage, cachePolicyJSON []byte) (storage caches.StorageInterface, shouldStop bool, err error) {
cachePolicy := &serverconfigs.HTTPCachePolicy{}

View File

@@ -7,13 +7,14 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/ratelimit"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/iwind/TeaGo/types"
"net"
"os"
"strings"
"sync"
"sync/atomic"
"time"
@@ -21,19 +22,20 @@ import (
// ClientConn 客户端连接
type ClientConn struct {
once sync.Once
globalLimiter *ratelimit.Counter
once sync.Once
isTLS bool
hasDeadline bool
hasRead bool
isLO bool // 是否为环路
hasResetSYNFlood bool
BaseClientConn
}
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool, globalLimiter *ratelimit.Counter) net.Conn {
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
if quickClose {
// TCP
tcpConn, ok := conn.(*net.TCPConn)
@@ -43,10 +45,28 @@ func NewClientConn(conn net.Conn, isTLS bool, quickClose bool, globalLimiter *ra
}
}
return &ClientConn{BaseClientConn: BaseClientConn{rawConn: conn}, isTLS: isTLS, globalLimiter: globalLimiter}
// 是否为环路
var remoteAddr = conn.RemoteAddr().String()
var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
return &ClientConn{
BaseClientConn: BaseClientConn{rawConn: conn},
isTLS: isTLS,
isLO: isLO,
}
}
func (this *ClientConn) Read(b []byte) (n int, err error) {
// 环路直接读取
if this.isLO {
n, err = this.rawConn.Read(b)
if n > 0 {
atomic.AddUint64(&teaconst.InTrafficBytes, uint64(n))
}
return
}
// TLS
if this.isTLS {
if !this.hasDeadline {
_ = this.rawConn.SetReadDeadline(time.Now().Add(time.Duration(nodeconfigs.DefaultTLSHandshakeTimeout) * time.Second)) // TODO 握手超时时间可以设置
@@ -57,6 +77,7 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
}
}
// 开始读取
n, err = this.rawConn.Read(b)
if n > 0 {
atomic.AddUint64(&teaconst.InTrafficBytes, uint64(n))
@@ -65,18 +86,21 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
}
}
// SYN Flood检测
var isHandshakeError = err != nil && os.IsTimeout(err) && !this.hasRead
if isHandshakeError {
_ = this.SetLinger(0)
}
var synFloodConfig = sharedNodeConfig.SYNFloodConfig()
if synFloodConfig != nil && synFloodConfig.IsOn {
if isHandshakeError {
this.increaseSYNFlood(synFloodConfig)
} else if err == nil && !this.hasResetSYNFlood {
this.hasResetSYNFlood = true
this.resetSYNFlood()
// SYN Flood检测
if this.serverId == 0 || !this.hasResetSYNFlood {
var synFloodConfig = sharedNodeConfig.SYNFloodConfig()
if synFloodConfig != nil && synFloodConfig.IsOn {
if isHandshakeError {
this.increaseSYNFlood(synFloodConfig)
} else if err == nil && !this.hasResetSYNFlood {
this.hasResetSYNFlood = true
this.resetSYNFlood()
}
}
}
@@ -87,7 +111,15 @@ func (this *ClientConn) Write(b []byte) (n int, err error) {
n, err = this.rawConn.Write(b)
if n > 0 {
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
// 统计当前服务带宽
if this.serverId > 0 {
if !this.isLO { // 环路不统计带宽,避免缓存预热等行为产生带宽
stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
}
}
}
return
}
@@ -96,15 +128,10 @@ func (this *ClientConn) Close() error {
err := this.rawConn.Close()
// 全局并发数限制
this.once.Do(func() {
if this.globalLimiter != nil {
this.globalLimiter.Release()
}
})
// 单个服务并发数限制
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
if this.hasLimit {
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
}
return err
}
@@ -137,7 +164,7 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
var ip = this.RawIP()
if len(ip) > 0 && !iplibrary.IsInWhiteList(ip) && (!synFloodConfig.IgnoreLocal || !utils.IsLocalIP(ip)) {
var timestamp = utils.NextMinuteUnixTime()
var result = ttlcache.SharedCache.IncreaseInt64("SYN_FLOOD:"+ip, 1, timestamp)
var result = ttlcache.SharedCache.IncreaseInt64("SYN_FLOOD:"+ip, 1, timestamp, true)
var minAttempts = synFloodConfig.MinAttempts
if minAttempts < 5 {
minAttempts = 5

View File

@@ -8,8 +8,10 @@ type BaseClientConn struct {
rawConn net.Conn
isBound bool
userId int64
serverId int64
remoteAddr string
hasLimit bool
isClosed bool
}
@@ -31,11 +33,32 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
this.isBound = true
this.serverId = serverId
this.remoteAddr = remoteAddr
this.hasLimit = true
// 检查是否可以连接
return sharedClientConnLimiter.Add(this.rawConn.RemoteAddr().String(), serverId, remoteAddr, maxConnsPerServer, maxConnsPerIP)
}
// SetServerId 设置服务ID
func (this *BaseClientConn) SetServerId(serverId int64) {
this.serverId = serverId
}
// ServerId 读取当前连接绑定的服务ID
func (this *BaseClientConn) ServerId() int64 {
return this.serverId
}
// SetUserId 设置所属服务的用户ID
func (this *BaseClientConn) SetUserId(userId int64) {
this.userId = userId
}
// UserId 获取当前连接所属服务的用户ID
func (this *BaseClientConn) UserId() int64 {
return this.userId
}
// RawIP 原本IP
func (this *BaseClientConn) RawIP() string {
ip, _, _ := net.SplitHostPort(this.rawConn.RemoteAddr().String())

View File

@@ -11,4 +11,16 @@ type ClientConnInterface interface {
// Bind 绑定服务
Bind(serverId int64, remoteAddr string, maxConnsPerServer int, maxConnsPerIP int) bool
// ServerId 获取服务ID
ServerId() int64
// SetServerId 设置服务ID
SetServerId(serverId int64)
// SetUserId 设置所属服务的用户ID
SetUserId(userId int64)
// UserId 获取当前连接所属服务的用户ID
UserId() int64
}

View File

@@ -3,16 +3,13 @@
package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/ratelimit"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"net"
)
var sharedConnectionsLimiter = ratelimit.NewCounter(nodeconfigs.DefaultTCPMaxConnections)
// ClientListener 客户端网络监听
type ClientListener struct {
rawListener net.Listener
@@ -36,13 +33,8 @@ func (this *ClientListener) IsTLS() bool {
}
func (this *ClientListener) Accept() (net.Conn, error) {
// 限制并发连接数
var limiter = sharedConnectionsLimiter
limiter.Ack()
conn, err := this.rawListener.Accept()
if err != nil {
limiter.Release()
return nil, err
}
@@ -50,22 +42,30 @@ func (this *ClientListener) Accept() (net.Conn, error) {
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err == nil {
canGoNext, _ := iplibrary.AllowIP(ip, 0)
var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
if !canGoNext ||
(!waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)) {
if !canGoNext || beingDenied {
tcpConn, ok := conn.(*net.TCPConn)
if ok {
_ = tcpConn.SetLinger(0)
}
_ = conn.Close()
limiter.Release()
// 使用本地防火墙延长封禁
if beingDenied {
var fw = firewalls.Firewall()
if fw != nil && !fw.IsMock() {
_ = fw.DropSourceIP(ip, 60)
}
}
return this.Accept()
}
}
return NewClientConn(conn, this.isTLS, this.quickClose, limiter), nil
return NewClientConn(conn, this.isTLS, this.quickClose), nil
}
func (this *ClientListener) Close() error {

View File

@@ -0,0 +1,7 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
type LingerConn interface {
SetLinger(sec int) error
}

View File

@@ -80,6 +80,13 @@ Loop:
return nil
}
// 发送到本地
if sharedHTTPAccessLogViewer.HasConns() {
for _, accessLog := range accessLogs {
sharedHTTPAccessLogViewer.Send(accessLog)
}
}
// 发送到API
if this.rpcClient == nil {
client, err := rpc.SharedRPC()

View File

@@ -0,0 +1,116 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
"net"
"os"
"sync"
"sync/atomic"
)
var sharedHTTPAccessLogViewer = NewHTTPAccessLogViewer()
// HTTPAccessLogViewer 本地访问日志浏览器
type HTTPAccessLogViewer struct {
sockFile string
listener net.Listener
connMap map[int64]net.Conn // connId => net.Conn
connId int64
locker sync.Mutex
}
// NewHTTPAccessLogViewer 获取新对象
func NewHTTPAccessLogViewer() *HTTPAccessLogViewer {
return &HTTPAccessLogViewer{
sockFile: os.TempDir() + "/" + teaconst.AccessLogSockName,
connMap: map[int64]net.Conn{},
}
}
// Start 启动
func (this *HTTPAccessLogViewer) Start() error {
this.locker.Lock()
defer this.locker.Unlock()
if this.listener == nil {
// remove if exists
_ = os.Remove(this.sockFile)
// start listening
listener, err := net.Listen("unix", this.sockFile)
if err != nil {
return err
}
this.listener = listener
go func() {
for {
conn, err := this.listener.Accept()
if err != nil {
remotelogs.Error("ACCESS_LOG", "start local reading failed: "+err.Error())
break
}
this.locker.Lock()
var connId = this.nextConnId()
this.connMap[connId] = conn
go func() {
this.startReading(conn, connId)
}()
this.locker.Unlock()
}
}()
}
return nil
}
// HasConns 检查是否有连接
func (this *HTTPAccessLogViewer) HasConns() bool {
this.locker.Lock()
defer this.locker.Unlock()
return len(this.connMap) > 0
}
// Send 发送日志
func (this *HTTPAccessLogViewer) Send(accessLog *pb.HTTPAccessLog) {
var conns = []net.Conn{}
this.locker.Lock()
for _, conn := range this.connMap {
conns = append(conns, conn)
}
this.locker.Unlock()
if len(conns) == 0 {
return
}
for _, conn := range conns {
// ignore error
_, _ = conn.Write([]byte(accessLog.RemoteAddr + " [" + accessLog.TimeLocal + "] \"" + accessLog.RequestMethod + " " + accessLog.Scheme + "://" + accessLog.Host + accessLog.RequestURI + " " + accessLog.Proto + "\" " + types.String(accessLog.Status) + " - " + fmt.Sprintf("%.2fms", accessLog.RequestTime*1000) + "\n"))
}
}
func (this *HTTPAccessLogViewer) nextConnId() int64 {
return atomic.AddInt64(&this.connId, 1)
}
func (this *HTTPAccessLogViewer) startReading(conn net.Conn, connId int64) {
var buf = make([]byte, 1024)
for {
_, err := conn.Read(buf)
if err != nil {
this.locker.Lock()
delete(this.connMap, connId)
this.locker.Unlock()
break
}
}
}

View File

@@ -0,0 +1,247 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"context"
"crypto/tls"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"io"
"io/ioutil"
"net"
"net/http"
"regexp"
"strings"
"time"
)
func init() {
events.On(events.EventStart, func() {
goman.New(func() {
SharedHTTPCacheTaskManager.Start()
})
})
}
var SharedHTTPCacheTaskManager = NewHTTPCacheTaskManager()
// HTTPCacheTaskManager 缓存任务管理
type HTTPCacheTaskManager struct {
ticker *time.Ticker
httpClient *http.Client
protocolReg *regexp.Regexp
taskQueue chan *pb.PurgeServerCacheRequest
}
func NewHTTPCacheTaskManager() *HTTPCacheTaskManager {
var duration = 30 * time.Second
if Tea.IsTesting() {
duration = 10 * time.Second
}
return &HTTPCacheTaskManager{
ticker: time.NewTicker(duration),
httpClient: &http.Client{
Timeout: 10 * time.Minute, // TODO 可以设置请求超时时间
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
_, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
return net.Dial(network, "127.0.0.1:"+port)
},
MaxIdleConns: 128,
MaxIdleConnsPerHost: 32,
MaxConnsPerHost: 32,
IdleConnTimeout: 2 * time.Minute,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 0,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
},
protocolReg: regexp.MustCompile(`^(?i)(http|https)://`),
taskQueue: make(chan *pb.PurgeServerCacheRequest, 1024),
}
}
func (this *HTTPCacheTaskManager) Start() {
// task queue
goman.New(func() {
rpcClient, _ := rpc.SharedRPC()
if rpcClient != nil {
for taskReq := range this.taskQueue {
_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
if err != nil {
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
}
}
}
})
// Loop
for range this.ticker.C {
err := this.Loop()
if err != nil {
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "execute task failed: "+err.Error())
}
}
}
func (this *HTTPCacheTaskManager) Loop() error {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
if err != nil {
return err
}
var keys = resp.HttpCacheTaskKeys
if len(keys) == 0 {
return nil
}
var pbResults = []*pb.UpdateHTTPCacheTaskKeysStatusRequest_KeyResult{}
for _, key := range keys {
err = this.processKey(key)
var pbResult = &pb.UpdateHTTPCacheTaskKeysStatusRequest_KeyResult{
Id: key.Id,
NodeClusterId: key.NodeClusterId,
Error: "",
}
if err != nil {
pbResult.Error = err.Error()
}
pbResults = append(pbResults, pbResult)
}
_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
if err != nil {
return err
}
return nil
}
func (this *HTTPCacheTaskManager) PushTaskKeys(keys []string) {
select {
case this.taskQueue <- &pb.PurgeServerCacheRequest{
Keys: keys,
Prefixes: nil,
}:
default:
}
}
func (this *HTTPCacheTaskManager) processKey(key *pb.HTTPCacheTaskKey) error {
switch key.Type {
case "purge":
var storages = caches.SharedManager.FindAllStorages()
for _, storage := range storages {
switch key.KeyType {
case "key":
var cacheKeys = []string{key.Key}
if strings.HasPrefix(key.Key, "http://") {
cacheKeys = append(cacheKeys, strings.Replace(key.Key, "http://", "https://", 1))
} else if strings.HasPrefix(key.Key, "https://") {
cacheKeys = append(cacheKeys, strings.Replace(key.Key, "https://", "http://", 1))
}
// TODO 提升效率
for _, cacheKey := range cacheKeys {
var subKeys = []string{
cacheKey,
cacheKey + caches.SuffixMethod + "HEAD",
cacheKey + caches.SuffixWebP,
cacheKey + caches.SuffixPartial,
}
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
subKeys = append(subKeys, cacheKey+caches.SuffixCompression+encoding)
subKeys = append(subKeys, cacheKey+caches.SuffixWebP+caches.SuffixCompression+encoding)
}
err := storage.Purge(subKeys, "file")
if err != nil {
return err
}
}
case "prefix":
var prefixes = []string{key.Key}
if strings.HasPrefix(key.Key, "http://") {
prefixes = append(prefixes, strings.Replace(key.Key, "http://", "https://", 1))
} else if strings.HasPrefix(key.Key, "https://") {
prefixes = append(prefixes, strings.Replace(key.Key, "https://", "http://", 1))
}
err := storage.Purge(prefixes, "dir")
if err != nil {
return err
}
}
}
case "fetch":
err := this.fetchKey(key)
if err != nil {
return err
}
default:
return errors.New("invalid operation type '" + key.Type + "'")
}
return nil
}
// TODO 增加失败重试
// TODO 使用并发操作
func (this *HTTPCacheTaskManager) fetchKey(key *pb.HTTPCacheTaskKey) error {
var fullKey = key.Key
if !this.protocolReg.MatchString(fullKey) {
fullKey = "https://" + fullKey
}
req, err := http.NewRequest(http.MethodGet, fullKey, nil)
if err != nil {
return errors.New("invalid url: " + fullKey + ": " + err.Error())
}
// TODO 可以在管理界面自定义Header
req.Header.Set("X-Edge-Cache-Action", "fetch")
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36") // TODO 可以定义
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
resp, err := this.httpClient.Do(req)
if err != nil {
return errors.New("request failed: " + fullKey + ": " + err.Error())
}
defer func() {
_ = resp.Body.Close()
}()
// 读取内容,以便于生成缓存
_, _ = io.Copy(ioutil.Discard, resp.Body)
// 处理502
if resp.StatusCode == http.StatusBadGateway {
return errors.New("read origin site timeout")
}
return nil
}

View File

@@ -0,0 +1,25 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes_test
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/nodes"
"testing"
)
func TestHTTPCacheTaskManager_Loop(t *testing.T) {
// initialize cache policies
config, err := nodeconfigs.SharedNodeConfig()
if err != nil {
t.Fatal(err)
}
caches.SharedManager.UpdatePolicies(config.HTTPCachePolicies)
var manager = nodes.NewHTTPCacheTaskManager()
err = manager.Loop()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -22,16 +22,18 @@ var SharedHTTPClientPool = NewHTTPClientPool()
// HTTPClientPool 客户端池
type HTTPClientPool struct {
clientExpiredDuration time.Duration
clientsMap map[string]*HTTPClient // backend key => client
locker sync.Mutex
clientsMap map[string]*HTTPClient // backend key => client
cleanTicker *time.Ticker
locker sync.RWMutex
}
// NewHTTPClientPool 获取新对象
func NewHTTPClientPool() *HTTPClientPool {
var pool = &HTTPClientPool{
clientExpiredDuration: 3600 * time.Second,
clientsMap: map[string]*HTTPClient{},
cleanTicker: time.NewTicker(1 * time.Hour),
clientsMap: map[string]*HTTPClient{},
}
goman.New(func() {
@@ -53,10 +55,20 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
var key = origin.UniqueKey() + "@" + originAddr
this.locker.RLock()
client, found := this.clientsMap[key]
this.locker.RUnlock()
if found {
client.UpdateAccessTime()
return client.RawClient(), nil
}
// 这里不能使用RLock避免因为并发生成多个同样的client实例
this.locker.Lock()
defer this.locker.Unlock()
client, found := this.clientsMap[key]
// 再次查找
client, found = this.clientsMap[key]
if found {
client.UpdateAccessTime()
return client.RawClient(), nil
@@ -104,39 +116,41 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
}
}
var transport = &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
// 支持TOA的连接
conn, err := this.handleTOA(req, ctx, network, originAddr, connectionTimeout)
if conn != nil || err != nil {
return conn, err
}
var transport = &HTTPClientTransport{
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
// 支持TOA的连接
conn, err := this.handleTOA(req, ctx, network, originAddr, connectionTimeout)
if conn != nil || err != nil {
return conn, err
}
// 普通的连接
conn, err = (&net.Dialer{
Timeout: connectionTimeout,
KeepAlive: 1 * time.Minute,
}).DialContext(ctx, network, originAddr)
if err != nil {
return nil, err
}
// 普通的连接
conn, err = (&net.Dialer{
Timeout: connectionTimeout,
KeepAlive: 1 * time.Minute,
}).DialContext(ctx, network, originAddr)
if err != nil {
return nil, err
}
// 处理PROXY protocol
err = this.handlePROXYProtocol(conn, req, proxyProtocol)
if err != nil {
return nil, err
}
// 处理PROXY protocol
err = this.handlePROXYProtocol(conn, req, proxyProtocol)
if err != nil {
return nil, err
}
return conn, nil
return conn, nil
},
MaxIdleConns: 0,
MaxIdleConnsPerHost: idleConns,
MaxConnsPerHost: maxConnections,
IdleConnTimeout: idleTimeout,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 5 * time.Second,
TLSClientConfig: tlsConfig,
Proxy: nil,
},
MaxIdleConns: 0,
MaxIdleConnsPerHost: idleConns,
MaxConnsPerHost: maxConnections,
IdleConnTimeout: idleTimeout,
ExpectContinueTimeout: 1 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
TLSClientConfig: tlsConfig,
Proxy: nil,
}
rawClient = &http.Client{
@@ -168,13 +182,12 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
// 清理不使用的Client
func (this *HTTPClientPool) cleanClients() {
var ticker = time.NewTicker(this.clientExpiredDuration)
for range ticker.C {
currentAt := time.Now().Unix()
for range this.cleanTicker.C {
var nowTime = time.Now().Unix()
this.locker.Lock()
for k, client := range this.clientsMap {
if client.AccessTime() < currentAt+86400 { // 超过 N 秒没有调用就关闭
if client.AccessTime() < nowTime+86400 { // 超过 N 秒没有调用就关闭
delete(this.clientsMap, k)
client.Close()
}

View File

@@ -0,0 +1,26 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"net/http"
)
const emptyHTTPLocation = "/$EmptyHTTPLocation$"
type HTTPClientTransport struct {
*http.Transport
}
func (this *HTTPClientTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := this.Transport.RoundTrip(req)
if err != nil {
return resp, err
}
// 检查在跳转相关状态中Location是否存在
if httpStatusIsRedirect(resp.StatusCode) && len(resp.Header.Get("Location")) == 0 {
resp.Header.Set("Location", emptyHTTPLocation)
}
return resp, nil
}

View File

@@ -89,7 +89,9 @@ type HTTPRequest struct {
firewallRuleSetId int64
firewallRuleId int64
firewallActions []string
tags []string
wafHasRequestBody bool
tags []string
logAttrs map[string]string
@@ -164,15 +166,26 @@ func (this *HTTPRequest) Do() {
return
}
// 处理健康检查
var isHealthCheck = false
var healthCheckKey = this.RawReq.Header.Get(serverconfigs.HealthCheckHeaderName)
if len(healthCheckKey) > 0 {
if this.doHealthCheck(healthCheckKey, &isHealthCheck) {
this.doEnd()
return
}
}
if !this.isLnRequest {
// 特殊URL处理
if len(this.rawURI) > 1 && this.rawURI[1] == '.' {
// ACME
// TODO 需要配置是否启用ACME检测
if strings.HasPrefix(this.rawURI, "/.well-known/acme-challenge/") {
this.doACME()
this.doEnd()
return
if this.doACME() {
this.doEnd()
return
}
}
}
@@ -190,6 +203,24 @@ func (this *HTTPRequest) Do() {
return
}
// UAM
if !isHealthCheck {
if this.web.UAM != nil {
if this.web.UAM.IsOn {
if this.doUAM() {
this.doEnd()
return
}
}
} else if this.ReqServer.UAM != nil && this.ReqServer.UAM.IsOn {
this.web.UAM = this.ReqServer.UAM
if this.doUAM() {
this.doEnd()
return
}
}
}
// WAF
if this.web.FirewallRef != nil && this.web.FirewallRef.IsOn {
if this.doWAFRequest() {
@@ -232,6 +263,12 @@ func (this *HTTPRequest) Do() {
// 开始调用
func (this *HTTPRequest) doBegin() {
// 是否找不到域名匹配
if this.ReqServer.Id == 0 {
this.doMismatch()
return
}
if !this.isLnRequest {
// 处理request limit
if this.web.RequestLimit != nil &&
@@ -255,23 +292,6 @@ func (this *HTTPRequest) doBegin() {
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
}
// 处理健康检查
var isHealthCheck = false
var healthCheckKey = this.RawReq.Header.Get(serverconfigs.HealthCheckHeaderName)
if len(healthCheckKey) > 0 {
if this.doHealthCheck(healthCheckKey, &isHealthCheck) {
return
}
}
// UAM
if !isHealthCheck && this.ReqServer.UAM != nil && this.ReqServer.UAM.IsOn {
if this.doUAM() {
this.doEnd()
return
}
}
// 跳转
if len(this.web.HostRedirects) > 0 {
if this.doHostRedirect() {
@@ -339,7 +359,7 @@ func (this *HTTPRequest) doEnd() {
// 流量统计
// TODO 增加是否开启开关
if this.ReqServer != nil {
if this.ReqServer != nil && this.ReqServer.Id > 0 {
var countCached int64 = 0
var cachedBytes int64 = 0
@@ -356,17 +376,17 @@ func (this *HTTPRequest) doEnd() {
}
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.SentBodyBytes()+this.writer.SentHeaderBytes(), cachedBytes, 1, countCached, countAttacks, attackBytes, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
}
// 指标
if metrics.SharedManager.HasHTTPMetrics() {
this.doMetricsResponse()
}
// 指标
if metrics.SharedManager.HasHTTPMetrics() {
this.doMetricsResponse()
}
// 统计
if this.web.StatRef != nil && this.web.StatRef.IsOn {
// 放到最后执行
this.doStat()
// 统计
if this.web.StatRef != nil && this.web.StatRef.IsOn {
// 放到最后执行
this.doStat()
}
}
}
@@ -521,6 +541,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
}
}
// UAM
if web.UAM != nil && (web.UAM.IsPrior || isTop) {
this.web.UAM = web.UAM
}
// 重写规则
if len(web.RewriteRefs) > 0 {
for index, ref := range web.RewriteRefs {
@@ -1033,7 +1058,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
}
// X-Forwarded-For
forwardedFor := this.RawReq.Header.Get("X-Forwarded-For")
var forwardedFor = this.RawReq.Header.Get("X-Forwarded-For")
if len(forwardedFor) > 0 {
commaIndex := strings.Index(forwardedFor, ",")
if commaIndex > 0 {
@@ -1089,7 +1114,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
// 获取请求的客户端地址列表
func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
// X-Forwarded-For
forwardedFor := this.RawReq.Header.Get("X-Forwarded-For")
var forwardedFor = this.RawReq.Header.Get("X-Forwarded-For")
if len(forwardedFor) > 0 {
commaIndex := strings.Index(forwardedFor, ",")
if commaIndex > 0 {
@@ -1114,13 +1139,16 @@ func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
}
// Remote-Addr
remoteAddr := this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
result = append(result, host)
} else {
result = append(result, remoteAddr)
{
var remoteAddr = this.RawReq.RemoteAddr
host, _, err := net.SplitHostPort(remoteAddr)
if err == nil {
result = append(result, host)
} else {
result = append(result, remoteAddr)
}
}
return
}
@@ -1447,7 +1475,12 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
header["X-Forwarded-For"] = []string{strings.Join(forwardedFor, ", ") + ", " + remoteAddr}
}
} else {
header["X-Forwarded-For"] = []string{remoteAddr}
var clientRemoteAddr = this.requestRemoteAddr(true)
if len(clientRemoteAddr) > 0 && clientRemoteAddr != remoteAddr {
header["X-Forwarded-For"] = []string{clientRemoteAddr + ", " + remoteAddr}
} else {
header["X-Forwarded-For"] = []string{remoteAddr}
}
}
}

View File

@@ -4,34 +4,36 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"net/http"
"path/filepath"
)
func (this *HTTPRequest) doACME() {
func (this *HTTPRequest) doACME() (shouldStop bool) {
// TODO 对请求进行校验,防止恶意攻击
token := filepath.Base(this.RawReq.URL.Path)
var token = filepath.Base(this.RawReq.URL.Path)
if token == "acme-challenge" || len(token) <= 32 {
this.writer.WriteHeader(http.StatusNotFound)
return
return false
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
remotelogs.Error("RPC", "[ACME]rpc failed: "+err.Error())
return
return false
}
keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
if err != nil {
remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
return
return false
}
if len(keyResp.Key) == 0 {
this.writer.WriteHeader(http.StatusNotFound)
} else {
this.writer.Header().Set("Content-Type", "text/plain")
_, _ = this.writer.WriteString(keyResp.Key)
return false
}
this.tags = append(this.tags, "ACME")
this.writer.Header().Set("Content-Type", "text/plain")
_, _ = this.writer.WriteString(keyResp.Key)
return true
}

View File

@@ -3,12 +3,9 @@ package nodes
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
@@ -33,11 +30,6 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return
}
// 判断是否在预热
if (strings.HasPrefix(this.RawReq.RemoteAddr, "127.") || strings.HasPrefix(this.RawReq.RemoteAddr, "[::1]")) && this.RawReq.Header.Get("X-Cache-Action") == "preheat" {
return
}
// 添加 X-Cache Header
var addStatusHeader = this.web.Cache.AddStatusHeader
if addStatusHeader {
@@ -89,6 +81,12 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return
}
// 是否正在Purge
var isPurging = this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey
if isPurging {
this.RawReq.Method = http.MethodGet
}
// 校验请求
if !this.cacheRef.MatchRequest(this.RawReq) {
this.cacheRef = nil
@@ -136,8 +134,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
this.writer.cacheStorage = storage
// 如果正在预热,则不读取缓存,等待下一个步骤重新生成
if (strings.HasPrefix(this.RawReq.RemoteAddr, "127.") || strings.HasPrefix(this.RawReq.RemoteAddr, "[::1]")) && this.RawReq.Header.Get("X-Edge-Cache-Action") == "fetch" {
return
}
// 判断是否在Purge
if this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey {
if isPurging {
this.varMapping["cache.status"] = "PURGE"
var subKeys = []string{
@@ -159,22 +162,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 通过API节点清除别节点上的的Key
// TODO 改为队列不需要每个请求都使用goroutine
goman.New(func() {
rpcClient, err := rpc.SharedRPC()
if err == nil {
for _, rpcServerService := range rpcClient.ServerRPCList() {
_, err = rpcServerService.PurgeServerCache(rpcClient.Context(), &pb.PurgeServerCacheRequest{
Domains: []string{this.ReqHost},
Keys: []string{key},
Prefixes: nil,
})
if err != nil {
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
}
}
}
})
SharedHTTPCacheTaskManager.PushTaskKeys([]string{key})
return true
}
@@ -248,6 +236,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if reader == nil {
reader, err = storage.OpenReader(key, useStale, false)
if err != nil && this.cacheRef.AllowPartialContent {
// 尝试读取分片的缓存内容
if len(rangeHeader) == 0 {
// 默认读取开头
rangeHeader = "bytes=0-"
}
pReader, ranges := this.tryPartialReader(storage, key, useStale, rangeHeader)
if pReader != nil {
isPartialCache = true
@@ -382,6 +375,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if !this.isLnRequest && !isPartialCache && len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.addExpiresHeader(reader.ExpiresAt())
this.writer.WriteHeader(http.StatusNotModified)
this.isCached = true
this.cacheRef = nil
@@ -393,6 +387,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
if !this.isLnRequest && !isPartialCache && len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.addExpiresHeader(reader.ExpiresAt())
this.writer.WriteHeader(http.StatusNotModified)
this.isCached = true
this.cacheRef = nil
@@ -582,10 +577,12 @@ func (this *HTTPRequest) addExpiresHeader(expiresAt int64) {
if this.cacheRef.ExpiresTime.Overwrite || len(this.writer.Header().Get("Expires")) == 0 {
if this.cacheRef.ExpiresTime.AutoCalculate {
this.writer.Header().Set("Expires", time.Unix(utils.GMTUnixTime(expiresAt), 0).Format("Mon, 2 Jan 2006 15:04:05")+" GMT")
this.writer.Header().Del("Cache-Control")
} else if this.cacheRef.ExpiresTime.Duration != nil {
var duration = this.cacheRef.ExpiresTime.Duration.Duration()
if duration > 0 {
this.writer.Header().Set("Expires", utils.GMTTime(time.Now().Add(duration)).Format("Mon, 2 Jan 2006 15:04:05")+" GMT")
this.writer.Header().Del("Cache-Control")
}
}
}

View File

@@ -21,8 +21,13 @@ func (this *HTTPRequest) doHealthCheck(key string, isHealthCheck *bool) (stop bo
}
*isHealthCheck = true
if !data.GetBool("accessLogIsOn") {
this.disableLog = true
}
if data.GetBool("onlyBasicRequest") {
return true
}
return
}

View File

@@ -2,9 +2,18 @@
package nodes
import "net/http"
import (
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"net/http"
)
func (this *HTTPRequest) doRequestLimit() (shouldStop bool) {
// 是否在全局名单中
_, isInAllowedList := iplibrary.AllowIP(this.RemoteAddr(), this.ReqServer.Id)
if isInAllowedList {
return false
}
// 检查请求Body尺寸
// TODO 处理分片提交的内容
if this.web.RequestLimit.MaxBodyBytes() > 0 &&
@@ -15,7 +24,7 @@ func (this *HTTPRequest) doRequestLimit() (shouldStop bool) {
// 设置连接相关参数
if this.web.RequestLimit.MaxConns > 0 || this.web.RequestLimit.MaxConnsPerIP > 0 {
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
if requestConn != nil {
clientConn, ok := requestConn.(ClientConnInterface)
if ok && !clientConn.IsBound() {

View File

@@ -149,8 +149,7 @@ func (this *HTTPRequest) log() {
}
// 请求Body
// TODO 考虑在被攻击时记录攻击的requestBody如果requestBody匹配规则的话但要考虑请求尺寸、数据库容量避免因为日志而导致服务不稳定
if ref != nil && ref.ContainsField(serverconfigs.HTTPAccessLogFieldRequestBody) {
if (ref != nil && ref.ContainsField(serverconfigs.HTTPAccessLogFieldRequestBody)) || this.wafHasRequestBody {
accessLog.RequestBody = this.requestBodyData
if len(accessLog.RequestBody) > AccessLogMaxRequestBodySize {

View File

@@ -0,0 +1,68 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeutils"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"net/http"
"time"
)
// 域名无匹配情况处理
func (this *HTTPRequest) doMismatch() {
// 是否为健康检查
var healthCheckKey = this.RawReq.Header.Get(serverconfigs.HealthCheckHeaderName)
if len(healthCheckKey) > 0 {
_, err := nodeutils.Base64DecodeMap(healthCheckKey)
if err == nil {
this.writer.WriteHeader(http.StatusOK)
return
}
}
// 是否已经在黑名单
var remoteIP = this.RemoteAddr()
if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
this.Close()
return
}
// 根据配置进行相应的处理
if sharedNodeConfig.GlobalConfig != nil && sharedNodeConfig.GlobalConfig.HTTPAll.MatchDomainStrictly {
// 检查cc
// TODO 可以在管理端配置是否开启以及最多尝试次数
if len(remoteIP) > 0 {
const maxAttempts = 100
if ttlcache.SharedCache.IncreaseInt64("MISMATCH_DOMAIN:"+remoteIP, int64(1), time.Now().Unix()+60, false) > maxAttempts {
// 在加入之前再次检查黑名单
if !waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP) {
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteIP, time.Now().Unix()+int64(3600), 0, true, 0, 0, "access mismatch domain '"+this.RawReq.Host+"' too frequently")
}
}
}
// 处理当前连接
var httpAllConfig = sharedNodeConfig.GlobalConfig.HTTPAll
var mismatchAction = httpAllConfig.DomainMismatchAction
if mismatchAction != nil && mismatchAction.Code == "page" {
if mismatchAction.Options != nil {
this.writer.Header().Set("Content-Type", "text/html; charset=utf-8")
this.writer.WriteHeader(mismatchAction.Options.GetInt("statusCode"))
_, _ = this.writer.Write([]byte(mismatchAction.Options.GetString("contentHTML")))
} else {
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
}
return
} else {
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
this.Close()
return
}
}
http.Error(this.writer, "404 page not found: '"+this.URL()+"'", http.StatusNotFound)
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/types"
"io"
"net/http"
"net/url"
@@ -77,7 +78,7 @@ func (this *HTTPRequest) doReverseProxy() {
// 处理Scheme
if origin.Addr == nil {
err := errors.New(this.URL() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
err := errors.New(this.URL() + ": Origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write50x(err, http.StatusBadGateway, true)
return
@@ -124,6 +125,18 @@ func (this *HTTPRequest) doReverseProxy() {
if origin.Addr.HostHasVariables() {
originAddr = this.Format(originAddr)
}
// 端口跟随
if origin.FollowPort {
var originHostIndex = strings.Index(originAddr, ":")
if originHostIndex < 0 {
var originErr = errors.New(this.URL() + ": Invalid origin address '" + originAddr + "', lacking port")
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", originErr.Error())
this.write50x(originErr, http.StatusBadGateway, true)
return
}
originAddr = originAddr[:originHostIndex+1] + types.String(this.requestServerPort())
}
this.originAddr = originAddr
// RequestHost
@@ -133,6 +146,12 @@ func (this *HTTPRequest) doReverseProxy() {
} else {
this.RawReq.Host = requestHost
}
// 是否移除端口
if this.reverseProxy.RequestHostExcludingPort {
this.RawReq.Host = utils.ParseAddrHost(this.RawReq.Host)
}
this.RawReq.URL.Host = this.RawReq.Host
} else if this.reverseProxy.RequestHostType == serverconfigs.RequestHostTypeOrigin {
// 源站主机名
@@ -144,9 +163,21 @@ func (this *HTTPRequest) doReverseProxy() {
}
this.RawReq.Host = hostname
// 是否移除端口
if this.reverseProxy.RequestHostExcludingPort {
this.RawReq.Host = utils.ParseAddrHost(this.RawReq.Host)
}
this.RawReq.URL.Host = this.RawReq.Host
} else {
this.RawReq.URL.Host = this.ReqHost
// 是否移除端口
if this.reverseProxy.RequestHostExcludingPort {
this.RawReq.Host = utils.ParseAddrHost(this.RawReq.Host)
this.RawReq.URL.Host = utils.ParseAddrHost(this.RawReq.URL.Host)
}
}
// 重组请求URL
@@ -172,14 +203,14 @@ func (this *HTTPRequest) doReverseProxy() {
// 判断是否为Websocket请求
if this.RawReq.Header.Get("Upgrade") == "websocket" {
this.doWebsocket()
this.doWebsocket(requestHost)
return
}
// 获取请求客户端
client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol, this.reverseProxy.FollowRedirects)
if err != nil {
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": "+err.Error())
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Create client failed: "+err.Error())
this.write50x(err, http.StatusBadGateway, true)
return
}
@@ -196,13 +227,13 @@ func (this *HTTPRequest) doReverseProxy() {
// 客户端取消请求,则不提示
httpErr, ok := err.(*url.Error)
if !ok {
SharedOriginStateManager.Fail(origin, this.reverseProxy, func() {
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
this.write50x(err, http.StatusBadGateway, true)
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+": Request origin server failed: "+err.Error())
} else if httpErr.Err != context.Canceled {
SharedOriginStateManager.Fail(origin, this.reverseProxy, func() {
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
if httpErr.Timeout() {
@@ -213,7 +244,7 @@ func (this *HTTPRequest) doReverseProxy() {
this.write50x(err, http.StatusBadGateway, true)
}
if httpErr.Err != io.EOF {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": "+err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
}
} else {
// 是否为客户端方面的错误
@@ -252,7 +283,7 @@ func (this *HTTPRequest) doReverseProxy() {
if this.doWAFResponse(resp) {
err = resp.Body.Close()
if err != nil {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": "+err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Closing Error (WAF): "+err.Error())
}
return
}
@@ -262,7 +293,7 @@ func (this *HTTPRequest) doReverseProxy() {
if len(this.web.Pages) > 0 && this.doPage(resp.StatusCode) {
err = resp.Body.Close()
if err != nil {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": "+err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Closing error (Page): "+err.Error())
}
return
}
@@ -282,16 +313,32 @@ func (this *HTTPRequest) doReverseProxy() {
// 替换Location中的源站地址
var locationHeader = resp.Header.Get("Location")
if len(locationHeader) > 0 {
locationURL, err := url.Parse(locationHeader)
if err == nil && (locationURL.Host == originAddr || strings.HasPrefix(originAddr, locationURL.Host+":")) {
locationURL.Host = this.ReqHost
if this.IsHTTP {
locationURL.Scheme = "http"
} else if this.IsHTTPS {
locationURL.Scheme = "https"
}
// 空Location处理
if locationHeader == emptyHTTPLocation {
resp.Header.Del("Location")
} else {
// 自动修正Location中的源站地址
locationURL, err := url.Parse(locationHeader)
if err == nil && locationURL.Host != this.ReqHost && (locationURL.Host == originAddr || strings.HasPrefix(originAddr, locationURL.Host+":")) {
locationURL.Host = this.ReqHost
resp.Header.Set("Location", locationURL.String())
var oldScheme = locationURL.Scheme
// 尝试和当前Scheme一致
if this.IsHTTP {
locationURL.Scheme = "http"
} else if this.IsHTTPS {
locationURL.Scheme = "https"
}
// 如果和当前URL一样则可能是http -> https防止无限循环
if locationURL.String() == this.URL() {
locationURL.Scheme = oldScheme
resp.Header.Set("Location", locationURL.String())
} else {
resp.Header.Set("Location", locationURL.String())
}
}
}
}
@@ -312,7 +359,12 @@ func (this *HTTPRequest) doReverseProxy() {
// 是否有内容
if resp.ContentLength == 0 && len(resp.TransferEncoding) == 0 {
// 即使内容为0也需要读取一次以便于触发相关事件
var buf = utils.BytePool4k.Get()
_, _ = io.CopyBuffer(this.writer, resp.Body, buf)
utils.BytePool4k.Put(buf)
_ = resp.Body.Close()
this.writer.SetOk()
return
}
@@ -343,13 +395,13 @@ func (this *HTTPRequest) doReverseProxy() {
var closeErr = resp.Body.Close()
if closeErr != nil {
if !this.canIgnore(closeErr) {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": "+closeErr.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Closing error: "+closeErr.Error())
}
}
if err != nil && err != io.EOF {
if !this.canIgnore(err) {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": "+err.Error())
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Writing error: "+err.Error())
this.addError(err)
}
}

View File

@@ -51,7 +51,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return true
}
rootDir := this.web.Root.Dir
var rootDir = this.web.Root.Dir
if this.web.Root.HasVariables() {
rootDir = this.Format(rootDir)
}
@@ -59,9 +59,9 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
rootDir = Tea.Root + Tea.DS + rootDir
}
requestPath := this.uri
var requestPath = this.uri
questionMarkIndex := strings.Index(this.uri, "?")
var questionMarkIndex = strings.Index(this.uri, "?")
if questionMarkIndex > -1 {
requestPath = this.uri[:questionMarkIndex]
}
@@ -75,7 +75,9 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
if err == nil {
requestPath = p
} else {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
}
}
@@ -92,8 +94,8 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
}
filename := strings.Replace(requestPath, "/", Tea.DS, -1)
filePath := ""
var filename = strings.Replace(requestPath, "/", Tea.DS, -1)
var filePath = ""
if len(filename) > 0 && filename[0:1] == Tea.DS {
filePath = rootDir + filename
} else {
@@ -113,7 +115,9 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return
} else {
this.write50x(err, http.StatusInternalServerError, true)
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
}
@@ -142,7 +146,9 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return
} else {
this.write50x(err, http.StatusInternalServerError, true)
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
}
@@ -152,24 +158,24 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
// 响应header
respHeader := this.writer.Header()
var respHeader = this.writer.Header()
// mime type
contentType := ""
var contentType = ""
if this.web.ResponseHeaderPolicy == nil || !this.web.ResponseHeaderPolicy.IsOn || !this.web.ResponseHeaderPolicy.ContainsHeader("CONTENT-TYPE") {
ext := filepath.Ext(filePath)
var ext = filepath.Ext(filePath)
if len(ext) > 0 {
mimeType := mime.TypeByExtension(ext)
if len(mimeType) > 0 {
semicolonIndex := strings.Index(mimeType, ";")
mimeTypeKey := mimeType
var semicolonIndex = strings.Index(mimeType, ";")
var mimeTypeKey = mimeType
if semicolonIndex > 0 {
mimeTypeKey = mimeType[:semicolonIndex]
}
if _, found := textMimeMap[mimeTypeKey]; found {
if this.web.Charset != nil && this.web.Charset.IsOn && len(this.web.Charset.Charset) > 0 {
charset := this.web.Charset.Charset
var charset = this.web.Charset.Charset
if this.web.Charset.IsUpper {
charset = strings.ToUpper(charset)
}
@@ -197,7 +203,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
// 支持 ETag
eTag := "\"e" + fmt.Sprintf("%0x", xxhash.Sum64String(filename+strconv.FormatInt(stat.ModTime().UnixNano(), 10)+strconv.FormatInt(stat.Size(), 10))) + "\""
var eTag = "\"e" + fmt.Sprintf("%0x", xxhash.Sum64String(filename+strconv.FormatInt(stat.ModTime().UnixNano(), 10)+strconv.FormatInt(stat.Size(), 10))) + "\""
if len(respHeader.Get("ETag")) == 0 {
respHeader.Set("ETag", eTag)
}
@@ -227,7 +233,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
// 支持Range
respHeader.Set("Accept-Ranges", "bytes")
ifRangeHeaders, ok := this.RawReq.Header["If-Range"]
supportRange := true
var supportRange = true
if ok {
supportRange = false
for _, v := range ifRangeHeaders {
@@ -244,7 +250,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
// 支持Range
var ranges = []rangeutils.Range{}
if supportRange {
contentRange := this.RawReq.Header.Get("Range")
var contentRange = this.RawReq.Header.Get("Range")
if len(contentRange) > 0 {
if fileSize == 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
@@ -277,7 +283,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
respHeader.Set("Content-Length", strconv.FormatInt(fileSize, 10))
}
reader, err := os.OpenFile(filePath, os.O_RDONLY, 0444)
fileReader, err := os.OpenFile(filePath, os.O_RDONLY, 0444)
if err != nil {
this.write50x(err, http.StatusInternalServerError, true)
return true
@@ -291,12 +297,16 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.cacheRef = nil // 不支持缓存
}
this.writer.Prepare(nil, fileSize, http.StatusOK, true)
var resp = &http.Response{
ContentLength: fileSize,
Body: fileReader,
StatusCode: http.StatusOK,
}
this.writer.Prepare(resp, fileSize, http.StatusOK, true)
pool := this.bytePool(fileSize)
buf := pool.Get()
var pool = this.bytePool(fileSize)
var buf = pool.Get()
defer func() {
_ = reader.Close()
pool.Put(buf)
}()
@@ -304,12 +314,14 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(types.String(fileSize)))
this.writer.WriteHeader(http.StatusPartialContent)
ok, err := httpRequestReadRange(reader, buf, ranges[0].Start(), ranges[0].End(), func(buf []byte, n int) error {
ok, err := httpRequestReadRange(resp.Body, buf, ranges[0].Start(), ranges[0].End(), func(buf []byte, n int) error {
_, err := this.writer.Write(buf[:n])
return err
})
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
if !ok {
@@ -318,7 +330,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return true
}
} else if len(ranges) > 1 {
boundary := httpRequestGenBoundary()
var boundary = httpRequestGenBoundary()
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
this.writer.WriteHeader(http.StatusPartialContent)
@@ -330,30 +342,38 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
_, err = this.writer.WriteString("\r\n--" + boundary + "\r\n")
}
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(types.String(fileSize)) + "\r\n")
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
if len(contentType) > 0 {
_, err = this.writer.WriteString("Content-Type: " + contentType + "\r\n\r\n")
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
}
ok, err := httpRequestReadRange(reader, buf, r.Start(), r.End(), func(buf []byte, n int) error {
ok, err := httpRequestReadRange(resp.Body, buf, r.Start(), r.End(), func(buf []byte, n int) error {
_, err := this.writer.Write(buf[:n])
return err
})
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
if !ok {
@@ -365,14 +385,17 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
_, err = this.writer.WriteString("\r\n--" + boundary + "--\r\n")
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
} else {
_, err = io.CopyBuffer(this.writer, reader, buf)
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
return true
}
}
@@ -400,7 +423,9 @@ func (this *HTTPRequest) findIndexFile(dir string) (filename string, stat os.Fil
if strings.Contains(index, "*") {
indexFiles, err := filepath.Glob(dir + Tea.DS + index)
if err != nil {
logs.Error(err)
if !this.canIgnore(err) {
logs.Error(err)
}
this.addError(err)
continue
}

View File

@@ -208,20 +208,3 @@ func httpAcceptEncoding(acceptEncodings string, encoding string) bool {
}
return false
}
// 分隔编码
func httpAcceptEncodings(acceptEncodings string) (encodings []string) {
if len(acceptEncodings) == 0 {
return
}
var pieces = strings.Split(acceptEncodings, ",")
for _, piece := range pieces {
var qualityIndex = strings.Index(piece, ";")
if qualityIndex >= 0 {
piece = piece[:qualityIndex]
}
encodings = append(encodings, strings.TrimSpace(piece))
}
return
}

View File

@@ -53,11 +53,21 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
return true
}
var forceLog = this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn && this.ReqServer.HTTPFirewallPolicy.Log != nil && this.ReqServer.HTTPFirewallPolicy.Log.IsOn
var forceLog = false
var forceLogRequestBody = false
var forceLogRegionDenying = false
if this.ReqServer.HTTPFirewallPolicy != nil &&
this.ReqServer.HTTPFirewallPolicy.IsOn &&
this.ReqServer.HTTPFirewallPolicy.Log != nil &&
this.ReqServer.HTTPFirewallPolicy.Log.IsOn {
forceLog = true
forceLogRequestBody = this.ReqServer.HTTPFirewallPolicy.Log.RequestBody
forceLogRegionDenying = this.ReqServer.HTTPFirewallPolicy.Log.RegionDenying
}
// 当前服务的独立设置
if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
blocked, breakChecking := this.checkWAFRequest(this.web.FirewallPolicy, forceLog)
blocked, breakChecking := this.checkWAFRequest(this.web.FirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying)
if blocked {
return true
}
@@ -68,7 +78,7 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
// 公用的防火墙设置
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
blocked, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy, forceLog)
blocked, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy, forceLog, forceLogRequestBody, forceLogRegionDenying)
if blocked {
return true
}
@@ -80,15 +90,21 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
return
}
func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, forceLog bool) (blocked bool, breakChecking bool) {
func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, forceLog bool, logRequestBody bool, logDenying bool) (blocked bool, breakChecking bool) {
// 检查配置是否为空
if firewallPolicy == nil || !firewallPolicy.IsOn || firewallPolicy.Inbound == nil || !firewallPolicy.Inbound.IsOn || firewallPolicy.Mode == firewallconfigs.FirewallModeBypass {
return
}
// 检查IP白名单
remoteAddrs := this.requestRemoteAddrs()
inbound := firewallPolicy.Inbound
var remoteAddrs []string
if len(this.remoteAddr) > 0 {
remoteAddrs = []string{this.remoteAddr}
} else {
remoteAddrs = this.requestRemoteAddrs()
}
var inbound = firewallPolicy.Inbound
if inbound == nil {
return
}
@@ -159,13 +175,17 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
// TODO 可以配置对封禁的处理方式等
// TODO 需要记录日志信息
this.firewallPolicyId = firewallPolicy.Id
this.writer.WriteHeader(http.StatusForbidden)
this.writer.Close()
// 停止日志
this.disableLog = true
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyCountry")
}
return true, false
}
@@ -173,15 +193,19 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
// 检查省份封禁
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
provinceId := iplibrary.SharedProvinceManager.Lookup(result.Province)
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
// TODO 可以配置对封禁的处理方式等
// TODO 需要记录日志信息
this.firewallPolicyId = firewallPolicy.Id
this.writer.WriteHeader(http.StatusForbidden)
this.writer.Close()
// 停止日志
this.disableLog = true
if !logDenying {
this.disableLog = true
} else {
this.tags = append(this.tags, "denyProvince")
}
return true, false
}
@@ -194,12 +218,15 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
}
// 规则测试
w := sharedWAFManager.FindWAF(firewallPolicy.Id)
w := waf.SharedWAFManager.FindWAF(firewallPolicy.Id)
if w == nil {
return
}
goNext, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer)
goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchRequest(this, this.writer)
if forceLog && logRequestBody && hasRequestBody {
this.wafHasRequestBody = true
}
if err != nil {
if !this.canIgnore(err) {
remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())
@@ -238,9 +265,15 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
}
// 当前服务的独立设置
var forceLog = this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn && this.ReqServer.HTTPFirewallPolicy.Log != nil && this.ReqServer.HTTPFirewallPolicy.Log.IsOn
var forceLog = false
var forceLogRequestBody = false
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn && this.ReqServer.HTTPFirewallPolicy.Log != nil && this.ReqServer.HTTPFirewallPolicy.Log.IsOn {
forceLog = true
forceLogRequestBody = this.ReqServer.HTTPFirewallPolicy.Log.RequestBody
}
if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog)
blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp, forceLog, forceLogRequestBody)
if blocked {
return true
}
@@ -248,7 +281,7 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
// 公用的防火墙设置
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog)
blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp, forceLog, forceLogRequestBody)
if blocked {
return true
}
@@ -256,17 +289,20 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
return
}
func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response, forceLog bool) (blocked bool) {
func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFirewallPolicy, resp *http.Response, forceLog bool, logRequestBody bool) (blocked bool) {
if firewallPolicy == nil || !firewallPolicy.IsOn || !firewallPolicy.Outbound.IsOn || firewallPolicy.Mode == firewallconfigs.FirewallModeBypass {
return
}
w := sharedWAFManager.FindWAF(firewallPolicy.Id)
w := waf.SharedWAFManager.FindWAF(firewallPolicy.Id)
if w == nil {
return
}
goNext, ruleGroup, ruleSet, err := w.MatchResponse(this, resp, this.writer)
goNext, hasRequestBody, ruleGroup, ruleSet, err := w.MatchResponse(this, resp, this.writer)
if forceLog && logRequestBody && hasRequestBody {
this.wafHasRequestBody = true
}
if err != nil {
if !this.canIgnore(err) {
remotelogs.Error("HTTP_REQUEST_WAF", this.rawURI+": "+err.Error())

View File

@@ -2,7 +2,6 @@ package nodes
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
"net/http"
@@ -10,7 +9,7 @@ import (
)
// 处理Websocket请求
func (this *HTTPRequest) doWebsocket() {
func (this *HTTPRequest) doWebsocket(requestHost string) {
if this.web.WebsocketRef == nil || !this.web.WebsocketRef.IsOn || this.web.Websocket == nil || !this.web.Websocket.IsOn {
this.writer.WriteHeader(http.StatusForbidden)
this.addError(errors.New("websocket have not been enabled yet"))
@@ -20,7 +19,7 @@ func (this *HTTPRequest) doWebsocket() {
// TODO 实现handshakeTimeout
// 校验来源
requestOrigin := this.RawReq.Header.Get("Origin")
var requestOrigin = this.RawReq.Header.Get("Origin")
if len(requestOrigin) > 0 {
u, err := url.Parse(requestOrigin)
if err == nil {
@@ -34,7 +33,7 @@ func (this *HTTPRequest) doWebsocket() {
// 设置指定的来源域
if !this.web.Websocket.RequestSameOrigin && len(this.web.Websocket.RequestOrigin) > 0 {
newRequestOrigin := this.web.Websocket.RequestOrigin
var newRequestOrigin = this.web.Websocket.RequestOrigin
if this.web.Websocket.RequestOriginHasVariables() {
newRequestOrigin = this.Format(newRequestOrigin)
}
@@ -42,11 +41,24 @@ func (this *HTTPRequest) doWebsocket() {
}
// TODO 增加N次错误重试重试的时候需要尝试不同的源站
originConn, err := OriginConnect(this.origin, this.RawReq.RemoteAddr)
originConn, _, err := OriginConnect(this.origin, this.requestServerPort(), this.RawReq.RemoteAddr, requestHost)
if err != nil {
this.write50x(err, http.StatusBadGateway, false)
// 增加失败次数
SharedOriginStateManager.Fail(this.origin, requestHost, this.reverseProxy, func() {
this.reverseProxy.ResetScheduling()
})
return
}
if !this.origin.IsOk {
SharedOriginStateManager.Success(this.origin, func() {
this.reverseProxy.ResetScheduling()
})
}
defer func() {
_ = originConn.Close()
}()
@@ -66,7 +78,7 @@ func (this *HTTPRequest) doWebsocket() {
_ = clientConn.Close()
}()
goman.New(func() {
go func() {
var buf = utils.BytePool4k.Get()
defer utils.BytePool4k.Put(buf)
for {
@@ -84,6 +96,6 @@ func (this *HTTPRequest) doWebsocket() {
}
_ = clientConn.Close()
_ = originConn.Close()
})
}()
_, _ = io.Copy(originConn, clientConn)
}

View File

@@ -312,9 +312,10 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
if !caches.CanIgnoreErr(err) {
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
this.Header().Set("X-Cache", "BYPASS, write cache failed")
} else {
this.Header().Set("X-Cache", "BYPASS, "+err.Error())
}
this.Header().Set("X-Cache", "BYPASS, too many requests")
return
}
this.cacheWriter = cacheWriter
@@ -448,7 +449,9 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
this.rawReader = cacheReader
cacheReader.OnFail(func(err error) {
_ = this.cacheWriter.Discard()
if this.cacheWriter != nil {
_ = this.cacheWriter.Discard()
}
this.cacheWriter = nil
})
cacheReader.OnEOF(func() {
@@ -506,7 +509,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
var contentEncoding = this.GetHeader("Content-Encoding")
switch contentEncoding {
case "gzip", "deflate", "br":
case "gzip", "deflate", "br", "zstd":
reader, err := compressions.NewReader(resp.Body, contentEncoding)
if err != nil {
return
@@ -544,7 +547,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
var contentEncoding = this.GetHeader("Content-Encoding")
if this.compressionConfig == nil || !this.compressionConfig.IsOn {
if lists.ContainsString([]string{"gzip", "deflate", "br"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
if lists.ContainsString([]string{"gzip", "deflate", "br", "zstd"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
reader, err := compressions.NewReader(resp.Body, contentEncoding)
if err != nil {
return
@@ -561,7 +564,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
}
// 如果已经有编码则不处理
if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !lists.ContainsString([]string{"gzip", "deflate", "br"}, contentEncoding)) {
if len(contentEncoding) > 0 && (!this.compressionConfig.DecompressData || !lists.ContainsString([]string{"gzip", "deflate", "br", "zstd"}, contentEncoding)) {
return
}
@@ -836,7 +839,7 @@ func (this *HTTPWriter) HeaderData() []byte {
return nil
}
resp := &http.Response{}
var resp = &http.Response{}
resp.Header = this.Header()
if this.statusCode == 0 {
this.statusCode = http.StatusOK
@@ -859,6 +862,70 @@ func (this *HTTPWriter) SetOk() {
// Close 关闭
func (this *HTTPWriter) Close() {
this.finishWebP()
this.finishRequest()
this.finishCache()
this.finishCompression()
// 统计
if this.sentBodyBytes == 0 {
this.sentBodyBytes = this.counterWriter.TotalBytes()
}
}
// Hijack Hijack
func (this *HTTPWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
hijack, ok := this.rawWriter.(http.Hijacker)
if ok {
return hijack.Hijack()
}
return
}
// Flush Flush
func (this *HTTPWriter) Flush() {
flusher, ok := this.rawWriter.(http.Flusher)
if ok {
flusher.Flush()
}
}
// DelayRead 是否延迟读取Reader
func (this *HTTPWriter) DelayRead() bool {
return this.delayRead
}
// 计算stale时长
func (this *HTTPWriter) calculateStaleLife() int {
var staleLife = 600 // TODO 可以在缓存策略里设置此时间
var staleConfig = this.req.web.Cache.Stale
if staleConfig != nil && staleConfig.IsOn {
// 从Header中读取stale-if-error
var isDefinedInHeader = false
if staleConfig.SupportStaleIfErrorHeader {
var cacheControl = this.GetHeader("Cache-Control")
var pieces = strings.Split(cacheControl, ",")
for _, piece := range pieces {
var eqIndex = strings.Index(piece, "=")
if eqIndex > 0 && strings.TrimSpace(piece[:eqIndex]) == "stale-if-error" {
// 这里预示着如果stale-if-error=0可以关闭stale功能
staleLife = types.Int(strings.TrimSpace(piece[eqIndex+1:]))
isDefinedInHeader = true
break
}
}
}
// 自定义
if !isDefinedInHeader && staleConfig.Life != nil {
staleLife = types.Int(staleConfig.Life.Duration().Seconds())
}
}
return staleLife
}
// 结束WebP
func (this *HTTPWriter) finishWebP() {
// 处理WebP
if this.webpIsEncoding {
var webpCacheWriter caches.Writer
@@ -919,6 +986,7 @@ func (this *HTTPWriter) Close() {
}
if err != nil {
// 发生了错误终止处理
return
}
@@ -948,7 +1016,7 @@ func (this *HTTPWriter) Close() {
//webpConfig.SetLossless(1)
webpConfig.SetQuality(f)
timeline := 0
var timeline = 0
for i, img := range gifImage.Image {
err = anim.AddFrame(img, timeline, webpConfig)
@@ -988,15 +1056,10 @@ func (this *HTTPWriter) Close() {
}
}
}
}
if this.writer != nil {
_ = this.writer.Close()
}
if this.rawReader != nil {
_ = this.rawReader.Close()
}
// 结束缓存相关处理
func (this *HTTPWriter) finishCache() {
// 缓存
if this.cacheWriter != nil {
if this.isOk && this.cacheIsFinished {
@@ -1054,7 +1117,10 @@ func (this *HTTPWriter) Close() {
}
}
}
}
// 结束压缩相关处理
func (this *HTTPWriter) finishCompression() {
if this.compressionCacheWriter != nil {
if this.isOk {
err := this.compressionCacheWriter.Close()
@@ -1075,59 +1141,15 @@ func (this *HTTPWriter) Close() {
_ = this.compressionCacheWriter.Discard()
}
}
}
if this.sentBodyBytes == 0 {
this.sentBodyBytes = this.counterWriter.TotalBytes()
// 最终关闭
func (this *HTTPWriter) finishRequest() {
if this.writer != nil {
_ = this.writer.Close()
}
if this.rawReader != nil {
_ = this.rawReader.Close()
}
}
// Hijack Hijack
func (this *HTTPWriter) Hijack() (conn net.Conn, buf *bufio.ReadWriter, err error) {
hijack, ok := this.rawWriter.(http.Hijacker)
if ok {
return hijack.Hijack()
}
return
}
// Flush Flush
func (this *HTTPWriter) Flush() {
flusher, ok := this.rawWriter.(http.Flusher)
if ok {
flusher.Flush()
}
}
// DelayRead 是否延迟读取Reader
func (this *HTTPWriter) DelayRead() bool {
return this.delayRead
}
// 计算stale时长
func (this *HTTPWriter) calculateStaleLife() int {
var staleLife = 600 // TODO 可以在缓存策略里设置此时间
var staleConfig = this.req.web.Cache.Stale
if staleConfig != nil && staleConfig.IsOn {
// 从Header中读取stale-if-error
var isDefinedInHeader = false
if staleConfig.SupportStaleIfErrorHeader {
var cacheControl = this.GetHeader("Cache-Control")
var pieces = strings.Split(cacheControl, ",")
for _, piece := range pieces {
var eqIndex = strings.Index(piece, "=")
if eqIndex > 0 && strings.TrimSpace(piece[:eqIndex]) == "stale-if-error" {
// 这里预示着如果stale-if-error=0可以关闭stale功能
staleLife = types.Int(strings.TrimSpace(piece[eqIndex+1:]))
isDefinedInHeader = true
break
}
}
}
// 自定义
if !isDefinedInHeader && staleConfig.Life != nil {
staleLife = types.Int(staleConfig.Life.Duration().Seconds())
}
}
return staleLife
}

View File

@@ -43,7 +43,7 @@ func (this *Listener) Listen() error {
if this.group == nil {
return nil
}
protocol := this.group.Protocol()
var protocol = this.group.Protocol()
if protocol.IsUDPFamily() {
return this.listenUDP()
}
@@ -54,7 +54,7 @@ func (this *Listener) listenTCP() error {
if this.group == nil {
return nil
}
protocol := this.group.Protocol()
var protocol = this.group.Protocol()
tcpListener, err := this.createTCPListener()
if err != nil {
@@ -153,7 +153,7 @@ func (this *Listener) Close() error {
// 创建TCP监听器
func (this *Listener) createTCPListener() (net.Listener, error) {
listenConfig := net.ListenConfig{
var listenConfig = net.ListenConfig{
Control: nil,
KeepAlive: 0,
}

View File

@@ -40,6 +40,10 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
return nil, err
}
if tlsPolicy == nil {
return nil, nil
}
tlsPolicy.CheckOCSP()
return tlsPolicy.TLSConfig(), nil
@@ -62,7 +66,7 @@ func (this *BaseListener) buildTLSConfig() *tls.Config {
// 根据域名匹配证书
func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.Certificate, error) {
group := this.Group
var group = this.Group
if group == nil {
return nil, nil, errors.New("no configure found")
@@ -107,7 +111,7 @@ func (this *BaseListener) matchSSL(domain string) (*sslconfigs.SSLPolicy, *tls.C
}
// 证书是否匹配
sslConfig := server.SSLPolicy()
var sslConfig = server.SSLPolicy()
cert, ok := sslConfig.MatchDomain(domain)
if ok {
return sslConfig, cert, nil
@@ -127,7 +131,7 @@ func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconf
return
}
matchDomainStrictly := sharedNodeConfig.GlobalConfig != nil && sharedNodeConfig.GlobalConfig.HTTPAll.MatchDomainStrictly
var matchDomainStrictly = sharedNodeConfig.GlobalConfig != nil && sharedNodeConfig.GlobalConfig.HTTPAll.MatchDomainStrictly
if sharedNodeConfig.GlobalConfig != nil &&
len(sharedNodeConfig.GlobalConfig.HTTPAll.DefaultDomain) > 0 &&
@@ -144,9 +148,9 @@ func (this *BaseListener) findNamedServer(name string) (serverConfig *serverconf
}
// 如果没有找到,则匹配到第一个
group := this.Group
currentServers := group.Servers()
countServers := len(currentServers)
var group = this.Group
var currentServers = group.Servers()
var countServers = len(currentServers)
if countServers == 0 {
return nil, ""
}

View File

@@ -139,10 +139,10 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
// TLS域名
if this.isIP(reqHost) {
if rawReq.TLS != nil {
serverName := rawReq.TLS.ServerName
var serverName = rawReq.TLS.ServerName
if len(serverName) > 0 {
// 端口
index := strings.LastIndex(reqHost, ":")
var index = strings.LastIndex(reqHost, ":")
if index >= 0 {
reqHost = serverName + reqHost[index:]
} else {
@@ -154,7 +154,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
// 防止空Host
if len(reqHost) == 0 {
ctx := rawReq.Context()
var ctx = rawReq.Context()
if ctx != nil {
addr := ctx.Value(http.LocalAddrContextKey)
if addr != nil {
@@ -170,42 +170,30 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
server, serverName := this.findNamedServer(domain)
if server == nil {
server = this.findServerWithCNAME(domain)
if server == nil {
// 严格匹配域名模式下,我们拒绝用户访问
if sharedNodeConfig.GlobalConfig != nil && sharedNodeConfig.GlobalConfig.HTTPAll.MatchDomainStrictly {
httpAllConfig := sharedNodeConfig.GlobalConfig.HTTPAll
mismatchAction := httpAllConfig.DomainMismatchAction
if mismatchAction != nil && mismatchAction.Code == "page" {
if mismatchAction.Options != nil {
rawWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
rawWriter.WriteHeader(mismatchAction.Options.GetInt("statusCode"))
_, _ = rawWriter.Write([]byte(mismatchAction.Options.GetString("contentHTML")))
} else {
http.Error(rawWriter, "404 page not found: '"+rawReq.URL.String()+"'", http.StatusNotFound)
}
return
} else {
hijacker, ok := rawWriter.(http.Hijacker)
if ok {
conn, _, _ := hijacker.Hijack()
if conn != nil {
_ = conn.Close()
return
}
}
}
}
http.Error(rawWriter, "404 page not found: '"+rawReq.URL.String()+"'", http.StatusNotFound)
return
// 增加默认的一个服务
server = this.emptyServer()
} else {
serverName = domain
}
} else if !server.CNameAsDomain && server.CNameDomain == domain {
server = this.emptyServer()
}
// 绑定连接
if server != nil && server.Id > 0 {
var requestConn = rawReq.Context().Value(HTTPConnContextKey)
if requestConn != nil {
clientConn, ok := requestConn.(ClientConnInterface)
if ok {
clientConn.SetServerId(server.Id)
clientConn.SetUserId(server.UserId)
}
}
}
// 包装新请求对象
req := &HTTPRequest{
var req = &HTTPRequest{
RawReq: rawReq,
RawWriter: rawWriter,
ReqServer: server,
@@ -220,6 +208,7 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
req.Do()
}
// 检查host是否为IP
func (this *HTTPListener) isIP(host string) bool {
// IPv6
if strings.Index(host, "[") > -1 {
@@ -234,3 +223,21 @@ func (this *HTTPListener) isIP(host string) bool {
return true
}
// 默认的访问日志
func (this *HTTPListener) emptyServer() *serverconfigs.ServerConfig {
var server = &serverconfigs.ServerConfig{
Type: serverconfigs.ServerTypeHTTPProxy,
}
var accessLogRef = serverconfigs.NewHTTPAccessLogRef()
// TODO 需要配置是否记录日志
accessLogRef.IsOn = true
accessLogRef.Fields = append([]int{}, serverconfigs.HTTPAccessLogDefaultFieldsCodes...)
server.Web = &serverconfigs.HTTPWebConfig{
IsOn: true,
AccessLogRef: accessLogRef,
}
return server
}

View File

@@ -257,6 +257,12 @@ func (this *ListenerManager) addToFirewalld(groupAddrs []string) {
return
}
// 检查状态
err = exec.Command(firewallCmd, "--state").Run()
if err != nil {
return
}
remotelogs.Println("FIREWALLD", "open ports automatically")
for _, port := range ports {
{

View File

@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/types"
"github.com/pires/go-proxyproto"
"net"
"strings"
@@ -18,14 +19,24 @@ type TCPListener struct {
BaseListener
Listener net.Listener
port int
}
func (this *TCPListener) Serve() error {
listener := this.Listener
var listener = this.Listener
if this.Group.IsTLS() {
listener = tls.NewListener(listener, this.buildTLSConfig())
}
// 获取分组端口
var groupAddr = this.Group.Addr()
var portIndex = strings.LastIndex(groupAddr, ":")
if portIndex >= 0 {
var port = groupAddr[portIndex+1:]
this.port = types.Int(port)
}
for {
conn, err := listener.Accept()
if err != nil {
@@ -52,14 +63,48 @@ func (this *TCPListener) Reload(group *serverconfigs.ServerAddressGroup) {
}
func (this *TCPListener) handleConn(conn net.Conn) error {
firstServer := this.Group.FirstServer()
if firstServer == nil {
var server = this.Group.FirstServer()
if server == nil {
return errors.New("no server available")
}
if firstServer.ReverseProxy == nil {
if server.ReverseProxy == nil {
return errors.New("no ReverseProxy configured for the server")
}
// 绑定连接和服务
clientConn, ok := conn.(ClientConnInterface)
if ok {
clientConn.SetServerId(server.Id)
clientConn.SetUserId(server.UserId)
} else {
tlsConn, ok := conn.(*tls.Conn)
if ok {
var internalConn = tlsConn.NetConn()
if internalConn != nil {
clientConn, ok = internalConn.(ClientConnInterface)
if ok {
clientConn.SetServerId(server.Id)
clientConn.SetUserId(server.UserId)
}
}
}
}
// 是否已达到流量限制
if this.reachedTrafficLimit() {
// 关闭连接
tcpConn, ok := conn.(LingerConn)
if ok {
_ = tcpConn.SetLinger(0)
}
_ = conn.Close()
// TODO 使用系统防火墙drop当前端口的数据包一段时间1分钟
// 不能使用阻止IP的方法因为边缘节点只上有可能还有别的代理服务
return nil
}
// 记录域名排行
tlsConn, ok := conn.(*tls.Conn)
var recordStat = false
@@ -67,17 +112,17 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
var serverName = tlsConn.ConnectionState().ServerName
if len(serverName) > 0 {
// 统计
stats.SharedTrafficStatManager.Add(firstServer.Id, serverName, 0, 0, 1, 0, 0, 0, firstServer.ShouldCheckTrafficLimit(), firstServer.PlanId())
stats.SharedTrafficStatManager.Add(server.Id, serverName, 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
recordStat = true
}
}
// 统计
if !recordStat {
stats.SharedTrafficStatManager.Add(firstServer.Id, "", 0, 0, 1, 0, 0, 0, firstServer.ShouldCheckTrafficLimit(), firstServer.PlanId())
stats.SharedTrafficStatManager.Add(server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
}
originConn, err := this.connectOrigin(firstServer.Id, firstServer.ReverseProxy, conn.RemoteAddr().String())
originConn, err := this.connectOrigin(server.Id, server.ReverseProxy, conn.RemoteAddr().String())
if err != nil {
return err
}
@@ -88,17 +133,17 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
}
// PROXY Protocol
if firstServer.ReverseProxy != nil &&
firstServer.ReverseProxy.ProxyProtocol != nil &&
firstServer.ReverseProxy.ProxyProtocol.IsOn &&
(firstServer.ReverseProxy.ProxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || firstServer.ReverseProxy.ProxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
if server.ReverseProxy != nil &&
server.ReverseProxy.ProxyProtocol != nil &&
server.ReverseProxy.ProxyProtocol.IsOn &&
(server.ReverseProxy.ProxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || server.ReverseProxy.ProxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
var remoteAddr = conn.RemoteAddr()
var transportProtocol = proxyproto.TCPv4
if strings.Contains(remoteAddr.String(), "[") {
transportProtocol = proxyproto.TCPv6
}
header := proxyproto.Header{
Version: byte(firstServer.ReverseProxy.ProxyProtocol.Version),
var header = proxyproto.Header{
Version: byte(server.ReverseProxy.ProxyProtocol.Version),
Command: proxyproto.PROXY,
TransportProtocol: transportProtocol,
SourceAddr: remoteAddr,
@@ -113,7 +158,7 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
// 从源站读取
goman.New(func() {
originBuffer := utils.BytePool16k.Get()
var originBuffer = utils.BytePool16k.Get()
defer func() {
utils.BytePool16k.Put(originBuffer)
}()
@@ -127,8 +172,8 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
}
// 记录流量
if firstServer != nil {
stats.SharedTrafficStatManager.Add(firstServer.Id, "", int64(n), 0, 0, 0, 0, 0, firstServer.ShouldCheckTrafficLimit(), firstServer.PlanId())
if server != nil {
stats.SharedTrafficStatManager.Add(server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
}
}
if err != nil {
@@ -139,11 +184,17 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
})
// 从客户端读取
clientBuffer := utils.BytePool16k.Get()
var clientBuffer = utils.BytePool16k.Get()
defer func() {
utils.BytePool16k.Put(clientBuffer)
}()
for {
// 是否已达到流量限制
if this.reachedTrafficLimit() {
closer()
return nil
}
n, err := conn.Read(clientBuffer)
if n > 0 {
_, err = originConn.Write(clientBuffer[:n])
@@ -166,25 +217,46 @@ func (this *TCPListener) Close() error {
return this.Listener.Close()
}
// 连接源站
func (this *TCPListener) connectOrigin(serverId int64, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
if reverseProxy == nil {
return nil, errors.New("no reverse proxy config")
}
retries := 3
var retries = 3
var addr string
for i := 0; i < retries; i++ {
origin := reverseProxy.NextOrigin(nil)
var origin = reverseProxy.NextOrigin(nil)
if origin == nil {
continue
}
conn, err = OriginConnect(origin, remoteAddr)
// 回源主机名
var requestHost = ""
if len(reverseProxy.RequestHost) > 0 {
requestHost = reverseProxy.RequestHost
}
if len(origin.RequestHost) > 0 {
requestHost = origin.RequestHost
}
conn, addr, err = OriginConnect(origin, this.port, remoteAddr, requestHost)
if err != nil {
remotelogs.ServerError(serverId, "TCP_LISTENER", "unable to connect origin: "+origin.Addr.Host+":"+origin.Addr.PortRange+": "+err.Error(), "", nil)
remotelogs.ServerError(serverId, "TCP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
continue
} else {
return
}
}
err = errors.New("no origin can be used")
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
return
}
// 检查是否已经达到流量限制
func (this *TCPListener) reachedTrafficLimit() bool {
var server = this.Group.FirstServer()
if server == nil {
return true
}
return server.TrafficLimitStatus != nil && server.TrafficLimitStatus.IsValid()
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/types"
"github.com/pires/go-proxyproto"
"net"
"strings"
@@ -25,11 +26,21 @@ type UDPListener struct {
reverseProxy *serverconfigs.ReverseProxyConfig
port int
isClosed bool
}
func (this *UDPListener) Serve() error {
firstServer := this.Group.FirstServer()
// 获取分组端口
var groupAddr = this.Group.Addr()
var portIndex = strings.LastIndex(groupAddr, ":")
if portIndex >= 0 {
var port = groupAddr[portIndex+1:]
this.port = types.Int(port)
}
var firstServer = this.Group.FirstServer()
if firstServer == nil {
return errors.New("no server available")
}
@@ -110,7 +121,7 @@ func (this *UDPListener) Reload(group *serverconfigs.ServerAddressGroup) {
this.Reset()
// 重置配置
firstServer := this.Group.FirstServer()
var firstServer = this.Group.FirstServer()
if firstServer == nil {
return
}
@@ -122,15 +133,16 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
return nil, errors.New("no reverse proxy config")
}
retries := 3
var retries = 3
var addr string
for i := 0; i < retries; i++ {
origin := reverseProxy.NextOrigin(nil)
var origin = reverseProxy.NextOrigin(nil)
if origin == nil {
continue
}
conn, err = OriginConnect(origin, remoteAddr.String())
conn, addr, err = OriginConnect(origin, this.port, remoteAddr.String(), "")
if err != nil {
remotelogs.ServerError(serverId, "UDP_LISTENER", "unable to connect origin: "+origin.Addr.Host+":"+origin.Addr.PortRange+": "+err.Error(), "", nil)
remotelogs.ServerError(serverId, "UDP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
continue
} else {
// PROXY Protocol
@@ -159,7 +171,7 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
return
}
}
err = errors.New("no origin can be used")
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
return
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/configs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
@@ -15,12 +16,12 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/TeaOSLab/EdgeNode/internal/metrics"
"github.com/TeaOSLab/EdgeNode/internal/ratelimit"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/stats"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/waf"
"github.com/andybalholm/brotli"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/lists"
@@ -115,6 +116,7 @@ func (this *Node) Start() {
this.checkDisk()
// 读取API配置
remotelogs.Println("NODE", "init config ...")
err = this.syncConfig(0)
if err != nil {
_, err := nodeconfigs.SharedNodeConfig()
@@ -368,6 +370,42 @@ func (this *Node) loop() error {
}
sharedNodeConfig.ParentNodes = parentNodes
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
IsOk: true,
Error: "",
})
if err != nil {
return err
}
case "ddosProtectionChanged":
resp, err := rpcClient.NodeRPC().FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
if err != nil {
return err
}
if len(resp.DdosProtectionJSON) == 0 {
if sharedNodeConfig != nil {
sharedNodeConfig.DDoSProtection = nil
}
} else {
var ddosProtectionConfig = &ddosconfigs.ProtectionConfig{}
err = json.Unmarshal(resp.DdosProtectionJSON, ddosProtectionConfig)
if err != nil {
return errors.New("decode DDoS protection config failed: " + err.Error())
}
if sharedNodeConfig != nil {
sharedNodeConfig.DDoSProtection = ddosProtectionConfig
}
err = firewalls.SharedDDoSProtectionManager.Apply(ddosProtectionConfig)
if err != nil {
// 不阻塞
remotelogs.Error("NODE", "apply DDoS protection failed: "+err.Error())
}
}
// 修改为已同步
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
NodeTaskId: task.Id,
@@ -396,7 +434,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
clusterErr := this.checkClusterConfig()
if clusterErr != nil {
if os.IsNotExist(clusterErr) {
return err
return errors.New("can not find config file 'configs/api.yaml'")
}
return errors.New("check cluster config failed: " + clusterErr.Error())
}
@@ -426,7 +464,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
return nil
}
configJSON := configResp.NodeJSON
var configJSON = configResp.NodeJSON
if configResp.IsCompressed {
var reader = brotli.NewReader(bytes.NewReader(configJSON))
var configBuf = &bytes.Buffer{}
@@ -445,7 +483,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
nodeConfigUpdatedAt = time.Now().Unix()
nodeConfig := &nodeconfigs.NodeConfig{}
var nodeConfig = &nodeconfigs.NodeConfig{}
err = json.Unmarshal(configJSON, nodeConfig)
if err != nil {
return errors.New("decode config failed: " + err.Error())
@@ -453,6 +491,15 @@ func (this *Node) syncConfig(taskVersion int64) error {
teaconst.NodeId = nodeConfig.Id
teaconst.NodeIdString = types.String(teaconst.NodeId)
// 检查时间是否一致
// 这个需要在 teaconst.NodeId 设置之后因为上报到API节点的时候需要节点ID
if configResp.Timestamp > 0 {
var timestampDelta = configResp.Timestamp - time.Now().Unix()
if timestampDelta > 60 || timestampDelta < -60 {
remotelogs.Error("NODE", "node timestamp ('"+types.String(time.Now().Unix())+"') is not same as api node ('"+types.String(configResp.Timestamp)+"'), please sync the time")
}
}
// 写入到文件中
err = nodeConfig.Save()
if err != nil {
@@ -721,7 +768,6 @@ func (this *Node) listenSock() error {
"ipConns": ipConns,
"serverConns": serverConns,
"total": sharedListenerManager.TotalActiveConnections(),
"limiter": sharedConnectionsLimiter.Len(),
},
})
case "dropIP":
@@ -780,6 +826,18 @@ func (this *Node) listenSock() error {
} else {
_ = cmd.ReplyOk()
}
case "accesslog":
err := sharedHTTPAccessLogViewer.Start()
if err != nil {
_ = cmd.Reply(&gosock.Command{
Code: "error",
Params: map[string]interface{}{
"message": "start failed: " + err.Error(),
},
})
} else {
_ = cmd.ReplyOk()
}
}
})
@@ -813,7 +871,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
}
// WAF策略
sharedWAFManager.UpdatePolicies(config.FindAllFirewallPolicies())
waf.SharedWAFManager.UpdatePolicies(config.FindAllFirewallPolicies())
iplibrary.SharedActionManager.UpdateActions(config.FirewallActions)
// 统计指标
@@ -845,17 +903,6 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
this.maxThreads = config.MaxThreads
}
// max tcp connections
if config.TCPMaxConnections <= 0 {
config.TCPMaxConnections = nodeconfigs.DefaultTCPMaxConnections
}
if config.TCPMaxConnections != sharedConnectionsLimiter.Count() {
remotelogs.Println("NODE", "[TCP]changed tcp max connections to '"+types.String(config.TCPMaxConnections)+"'")
sharedConnectionsLimiter.Close()
sharedConnectionsLimiter = ratelimit.NewCounter(config.TCPMaxConnections)
}
// timezone
var timeZone = config.TimeZone
if len(timeZone) == 0 {
@@ -878,6 +925,29 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
if config.ProductConfig != nil {
teaconst.GlobalProductName = config.ProductConfig.Name
}
// DNS resolver
if config.DNSResolver != nil {
var err error
switch config.DNSResolver.Type {
case nodeconfigs.DNSResolverTypeGoNative:
err = os.Setenv("GODEBUG", "netdns=go")
case nodeconfigs.DNSResolverTypeCGO:
err = os.Setenv("GODEBUG", "netdns=cgo")
default:
// 默认使用go原生
err = os.Setenv("GODEBUG", "netdns=go")
}
if err != nil {
remotelogs.Error("NODE", "[DNS_RESOLVER]set env failed: "+err.Error())
}
} else {
// 默认使用go原生
err := os.Setenv("GODEBUG", "netdns=go")
if err != nil {
remotelogs.Error("NODE", "[DNS_RESOLVER]set env failed: "+err.Error())
}
}
}
// reload server config

View File

@@ -109,6 +109,7 @@ func (this *NodeStatusExecutor) update() {
cacheSpaceTR.End()
status.UpdatedAt = time.Now().Unix()
status.Timestamp = status.UpdatedAt
// 发送数据
jsonData, err := json.Marshal(status)

View File

@@ -8,5 +8,7 @@ type OriginState struct {
CountFails int64
UpdatedAt int64
Config *serverconfigs.OriginConfig
Addr string
TLSHost string
ReverseProxy *serverconfigs.ReverseProxyConfig
}

View File

@@ -26,6 +26,10 @@ func init() {
})
}
const (
maxOriginStates = 512 // 最多可以监控的源站状态数量
)
// OriginStateManager 源站状态管理
type OriginStateManager struct {
stateMap map[int64]*OriginState // originId => *OriginState
@@ -44,12 +48,6 @@ func NewOriginStateManager() *OriginStateManager {
// Start 启动
func (this *OriginStateManager) Start() {
events.OnKey(events.EventReload, this, func() {
this.locker.Lock()
this.stateMap = map[int64]*OriginState{}
this.locker.Unlock()
})
if Tea.IsTesting() {
this.ticker = time.NewTicker(10 * time.Second)
}
@@ -69,7 +67,8 @@ func (this *OriginStateManager) Stop() {
// Loop 单次循环检查
func (this *OriginStateManager) Loop() error {
if sharedNodeConfig == nil {
var nodeConfig = sharedNodeConfig // 复制
if nodeConfig == nil {
return nil
}
@@ -80,12 +79,12 @@ func (this *OriginStateManager) Loop() error {
this.locker.Lock()
for originId, state := range this.stateMap {
// 检查Origin是否正在使用
config := sharedNodeConfig.FindOrigin(originId)
if config == nil || !config.IsOn {
var originConfig = nodeConfig.FindOrigin(originId)
if originConfig == nil || !originConfig.IsOn {
delete(this.stateMap, originId)
continue
}
state.Config = config
state.Config = originConfig
currentStates = append(currentStates, state)
}
this.locker.Unlock()
@@ -95,12 +94,12 @@ func (this *OriginStateManager) Loop() error {
}
var count = len(currentStates)
wg := &sync.WaitGroup{}
var wg = &sync.WaitGroup{}
wg.Add(count)
for _, state := range currentStates {
go func(state *OriginState) {
defer wg.Done()
conn, err := OriginConnect(state.Config, "")
conn, _, err := OriginConnect(state.Config, 0, "", state.TLSHost)
if err == nil {
_ = conn.Close()
@@ -123,7 +122,7 @@ func (this *OriginStateManager) Loop() error {
}
// Fail 添加失败的源站
func (this *OriginStateManager) Fail(origin *serverconfigs.OriginConfig, reverseProxy *serverconfigs.ReverseProxyConfig, callback func()) {
func (this *OriginStateManager) Fail(origin *serverconfigs.OriginConfig, tlsHost string, reverseProxy *serverconfigs.ReverseProxyConfig, callback func()) {
if origin == nil || origin.Id <= 0 {
return
}
@@ -137,13 +136,14 @@ func (this *OriginStateManager) Fail(origin *serverconfigs.OriginConfig, reverse
state.Config.IsOk = true
}
state.TLSHost = tlsHost
state.CountFails++
state.Config = origin
state.ReverseProxy = reverseProxy
state.UpdatedAt = timestamp
if origin.IsOk {
origin.IsOk = state.CountFails > 5 // 超过 N 次之后认为是异常
origin.IsOk = state.CountFails < 5 // 超过 N 次之后认为是异常
if !origin.IsOk {
if callback != nil {
@@ -152,12 +152,17 @@ func (this *OriginStateManager) Fail(origin *serverconfigs.OriginConfig, reverse
}
}
} else {
this.stateMap[origin.Id] = &OriginState{
CountFails: 1,
Config: origin,
ReverseProxy: reverseProxy,
UpdatedAt: timestamp,
// 同时最多监控 N 个源站地址
if len(this.stateMap) < maxOriginStates {
this.stateMap[origin.Id] = &OriginState{
CountFails: 1,
Config: origin,
TLSHost: tlsHost,
ReverseProxy: reverseProxy,
UpdatedAt: timestamp,
}
}
origin.IsOk = true
}
this.locker.Unlock()

View File

@@ -3,47 +3,54 @@ package nodes
import (
"crypto/tls"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/types"
"net"
"strconv"
)
// OriginConnect 连接源站
func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.Conn, error) {
func OriginConnect(origin *serverconfigs.OriginConfig, serverPort int, remoteAddr string, tlsHost string) (originConn net.Conn, originAddr string, err error) {
if origin.Addr == nil {
return nil, errors.New("origin server address should not be empty")
return nil, "", errors.New("origin server address should not be empty")
}
// 支持TOA的连接
// 这个条件很重要如果没有传递remoteAddr表示不使用TOA
if len(remoteAddr) > 0 {
toaConfig := sharedTOAManager.Config()
var toaConfig = sharedTOAManager.Config()
if toaConfig != nil && toaConfig.IsOn {
retries := 3
var retries = 3
for i := 1; i <= retries; i++ {
port := int(toaConfig.RandLocalPort())
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + remoteAddr)
var port = int(toaConfig.RandLocalPort())
err = sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + remoteAddr)
if err != nil {
remotelogs.Error("TOA", "add failed: "+err.Error())
} else {
dialer := net.Dialer{
var dialer = net.Dialer{
Timeout: origin.ConnTimeoutDuration(),
LocalAddr: &net.TCPAddr{
Port: port,
},
}
originAddr = origin.Addr.PickAddress()
// 端口跟随
if origin.FollowPort && serverPort > 0 {
originAddr = configutils.QuoteIP(origin.Addr.Host) + ":" + types.String(serverPort)
}
var conn net.Conn
switch origin.Addr.Protocol {
case "", serverconfigs.ProtocolTCP, serverconfigs.ProtocolHTTP:
// TODO 支持TCP4/TCP6
// TODO 支持指定特定网卡
// TODO Addr支持端口范围如果有多个端口时随机一个端口使用
conn, err = dialer.Dial("tcp", origin.Addr.Host+":"+origin.Addr.PortRange)
conn, err = dialer.Dial("tcp", originAddr)
case serverconfigs.ProtocolTLS, serverconfigs.ProtocolHTTPS:
// TODO 支持TCP4/TCP6
// TODO 支持指定特定网卡
// TODO Addr支持端口范围如果有多个端口时随机一个端口使用
var tlsConfig = &tls.Config{
InsecureSkipVerify: true,
@@ -58,29 +65,38 @@ func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.C
}
}
}
if len(tlsHost) > 0 {
tlsConfig.ServerName = tlsHost
}
conn, err = tls.DialWithDialer(&dialer, "tcp", origin.Addr.Host+":"+origin.Addr.PortRange, tlsConfig)
conn, err = tls.DialWithDialer(&dialer, "tcp", originAddr, tlsConfig)
}
// TODO 需要在合适的时机删除TOA记录
if err == nil || i == retries {
return conn, err
return conn, originAddr, err
}
}
}
}
}
originAddr = origin.Addr.PickAddress()
// 端口跟随
if origin.FollowPort && serverPort > 0 {
originAddr = configutils.QuoteIP(origin.Addr.Host) + ":" + types.String(serverPort)
}
switch origin.Addr.Protocol {
case "", serverconfigs.ProtocolTCP, serverconfigs.ProtocolHTTP:
// TODO 支持TCP4/TCP6
// TODO 支持指定特定网卡
// TODO Addr支持端口范围如果有多个端口时随机一个端口使用
return net.DialTimeout("tcp", origin.Addr.Host+":"+origin.Addr.PortRange, origin.ConnTimeoutDuration())
originConn, err = net.DialTimeout("tcp", originAddr, origin.ConnTimeoutDuration())
return originConn, originAddr, err
case serverconfigs.ProtocolTLS, serverconfigs.ProtocolHTTPS:
// TODO 支持TCP4/TCP6
// TODO 支持指定特定网卡
// TODO Addr支持端口范围如果有多个端口时随机一个端口使用
var tlsConfig = &tls.Config{
InsecureSkipVerify: true,
@@ -95,17 +111,22 @@ func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.C
}
}
}
return tls.Dial("tcp", origin.Addr.Host+":"+origin.Addr.PortRange, tlsConfig)
case serverconfigs.ProtocolUDP:
addr, err := net.ResolveUDPAddr("udp", origin.Addr.Host+":"+origin.Addr.PortRange)
if err != nil {
return nil, err
if len(tlsHost) > 0 {
tlsConfig.ServerName = tlsHost
}
return net.DialUDP("udp", nil, addr)
originConn, err = tls.Dial("tcp", originAddr, tlsConfig)
return originConn, originAddr, err
case serverconfigs.ProtocolUDP:
addr, err := net.ResolveUDPAddr("udp", originAddr)
if err != nil {
return nil, originAddr, err
}
originConn, err = net.DialUDP("udp", nil, addr)
return originConn, originAddr, err
}
// TODO 支持从Unix、Pipe、HTTP、HTTPS中读取数据
return nil, errors.New("invalid origin scheme '" + origin.Addr.Protocol.String() + "'")
return nil, originAddr, errors.New("invalid origin scheme '" + origin.Addr.Protocol.String() + "'")
}

View File

@@ -13,7 +13,6 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
stringutil "github.com/iwind/TeaGo/utils/string"
"os"
"os/exec"
@@ -64,7 +63,7 @@ func (this *UpgradeManager) Start() {
goman.New(func() {
err = this.restart()
if err != nil {
logs.Println("UPGRADE_MANAGER", err.Error())
remotelogs.Error("UPGRADE_MANAGER", err.Error())
}
})
}
@@ -204,12 +203,12 @@ func (this *UpgradeManager) unzip(zipPath string) error {
}
// 先改先前的可执行文件
err := os.Rename(target+"/bin/edge-node", target+"/bin/.edge-node.old")
err := os.Rename(target+"/bin/edge-node", target+"/bin/.edge-node.dist")
hasBackup := err == nil
defer func() {
if !isOk && hasBackup {
// 失败时还原
_ = os.Rename(target+"/bin/.edge-node.old", target+"/bin/edge-node")
_ = os.Rename(target+"/bin/.edge-node.dist", target+"/bin/edge-node")
}
}()

View File

@@ -136,6 +136,10 @@ func (this *Regexp) Match(s []byte) bool {
return this.rawRegexp.Match(s)
}
func (this *Regexp) FindStringSubmatch(s string) []string {
return this.rawRegexp.FindStringSubmatch(s)
}
// ParseKeywords 提取表达式中的关键词
func (this *Regexp) ParseKeywords(exp string) (keywords []string) {
if len(exp) == 0 {

View File

@@ -125,11 +125,36 @@ func BenchmarkRegexp_MatchString2(b *testing.B) {
func BenchmarkRegexp_MatchString_CaseSensitive(b *testing.B) {
var r = re.MustCompile("(abc|def|ghi)")
b.Log("keywords:", r.Keywords())
b.ResetTimer()
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}
}
func BenchmarkRegexp_MatchString_CaseSensitive2(b *testing.B) {
var r = regexp.MustCompile("(abc|def|ghi)")
b.ResetTimer()
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}
}
func BenchmarkRegexp_MatchString_VS_FindSubString1(b *testing.B) {
var r = re.MustCompile("(?i)(chrome)")
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = r.Raw().MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}
}
func BenchmarkRegexp_MatchString_VS_FindSubString2(b *testing.B) {
var r = re.MustCompile("(?i)(chrome)")
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = r.Raw().FindStringSubmatch("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}
}
func testCompareStrings(s1 []string, s2 []string) bool {
if len(s1) != len(s2) {
return false

View File

@@ -67,6 +67,10 @@ func (this *RPCClient) HTTPAccessLogRPC() pb.HTTPAccessLogServiceClient {
return pb.NewHTTPAccessLogServiceClient(this.pickConn())
}
func (this *RPCClient) HTTPCacheTaskKeyRPC() pb.HTTPCacheTaskKeyServiceClient {
return pb.NewHTTPCacheTaskKeyServiceClient(this.pickConn())
}
func (this *RPCClient) APINodeRPC() pb.APINodeServiceClient {
return pb.NewAPINodeServiceClient(this.pickConn())
}
@@ -115,22 +119,14 @@ func (this *RPCClient) ServerRPC() pb.ServerServiceClient {
return pb.NewServerServiceClient(this.pickConn())
}
func (this *RPCClient) ServerRPCList() []pb.ServerServiceClient {
this.locker.Lock()
defer this.locker.Unlock()
var clients = []pb.ServerServiceClient{}
for _, conn := range this.conns {
clients = append(clients, pb.NewServerServiceClient(conn))
}
return clients
}
func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient {
return pb.NewServerDailyStatServiceClient(this.pickConn())
}
func (this *RPCClient) ServerBandwidthStatRPC() pb.ServerBandwidthStatServiceClient {
return pb.NewServerBandwidthStatServiceClient(this.pickConn())
}
func (this *RPCClient) MetricStatRPC() pb.MetricStatServiceClient {
return pb.NewMetricStatServiceClient(this.pickConn())
}

View File

@@ -0,0 +1,152 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package stats
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"sync"
"time"
)
var SharedBandwidthStatManager = NewBandwidthStatManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedBandwidthStatManager.Start()
})
})
}
type BandwidthStat struct {
Day string
TimeAt string
UserId int64
ServerId int64
CurrentBytes int64
CurrentTimestamp int64
MaxBytes int64
}
// BandwidthStatManager 服务带宽统计
type BandwidthStatManager struct {
m map[string]*BandwidthStat // key => *BandwidthStat
lastTime string // 上一次执行的时间
ticker *time.Ticker
locker sync.Mutex
}
func NewBandwidthStatManager() *BandwidthStatManager {
return &BandwidthStatManager{
m: map[string]*BandwidthStat{},
ticker: time.NewTicker(1 * time.Minute), // 时间小于1分钟是为了更快速地上传结果
}
}
func (this *BandwidthStatManager) Start() {
for range this.ticker.C {
err := this.Loop()
if err != nil && !rpc.IsConnError(err) {
remotelogs.Error("BANDWIDTH_STAT_MANAGER", err.Error())
}
}
}
func (this *BandwidthStatManager) Loop() error {
var now = time.Now()
var day = timeutil.Format("Ymd", now)
var currentTime = timeutil.FormatTime("Hi", now.Unix()/300*300)
if this.lastTime == currentTime {
return nil
}
this.lastTime = currentTime
var pbStats = []*pb.ServerBandwidthStat{}
this.locker.Lock()
for key, stat := range this.m {
if stat.Day < day || stat.TimeAt < currentTime {
pbStats = append(pbStats, &pb.ServerBandwidthStat{
Id: 0,
UserId: stat.UserId,
ServerId: stat.ServerId,
Day: stat.Day,
TimeAt: stat.TimeAt,
Bytes: stat.MaxBytes,
})
delete(this.m, key)
}
}
this.locker.Unlock()
if len(pbStats) > 0 {
// 上传
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
_, err = rpcClient.ServerBandwidthStatRPC().UploadServerBandwidthStats(rpcClient.Context(), &pb.UploadServerBandwidthStatsRequest{ServerBandwidthStats: pbStats})
if err != nil {
return err
}
}
return nil
}
// Add 添加带宽数据
func (this *BandwidthStatManager) Add(userId int64, serverId int64, bytes int64) {
if serverId <= 0 || bytes == 0 {
return
}
var now = time.Now()
var timestamp = now.Unix()
var day = timeutil.Format("Ymd", now)
var timeAt = timeutil.FormatTime("Hi", now.Unix()/300*300)
var key = types.String(serverId) + "@" + day + "@" + timeAt
this.locker.Lock()
stat, ok := this.m[key]
if ok {
// 此刻如果发生用户IDuserId的变化也忽略等N分钟后有新记录后再换
if stat.CurrentTimestamp == timestamp {
stat.CurrentBytes += bytes
} else {
stat.CurrentBytes = bytes
stat.CurrentTimestamp = timestamp
}
if stat.CurrentBytes > stat.MaxBytes {
stat.MaxBytes = stat.CurrentBytes
}
} else {
this.m[key] = &BandwidthStat{
Day: day,
TimeAt: timeAt,
UserId: userId,
ServerId: serverId,
CurrentBytes: bytes,
MaxBytes: bytes,
CurrentTimestamp: timestamp,
}
}
this.locker.Unlock()
}
func (this *BandwidthStatManager) Inspect() {
this.locker.Lock()
logs.PrintAsJSON(this.m)
this.locker.Unlock()
}

View File

@@ -0,0 +1,33 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package stats_test
import (
"github.com/TeaOSLab/EdgeNode/internal/stats"
"testing"
"time"
)
func TestBandwidthStatManager_Add(t *testing.T) {
var manager = stats.NewBandwidthStatManager()
manager.Add(1, 1, 10)
manager.Add(1, 1, 10)
manager.Add(1, 1, 10)
time.Sleep(1 * time.Second)
manager.Add(1, 1, 15)
time.Sleep(1 * time.Second)
manager.Add(1, 1, 25)
manager.Add(1, 1, 75)
manager.Inspect()
}
func TestBandwidthStatManager_Loop(t *testing.T) {
var manager = stats.NewBandwidthStatManager()
manager.Add(1, 1, 10)
manager.Add(1, 1, 10)
manager.Add(1, 1, 10)
err := manager.Loop()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -123,7 +123,7 @@ func (this *HTTPRequestStatManager) AddRemoteAddr(serverId int64, remoteAddr str
if remoteAddr[0] == '[' { // 排除IPv6
return
}
index := strings.Index(remoteAddr, ":")
var index = strings.Index(remoteAddr, ":")
var ip string
if index < 0 {
ip = remoteAddr
@@ -177,18 +177,18 @@ func (this *HTTPRequestStatManager) AddFirewallRuleGroupId(serverId int64, firew
// Loop 单个循环
func (this *HTTPRequestStatManager) Loop() error {
timeout := time.NewTimer(10 * time.Minute) // 执行的最大时间
var timeout = time.NewTimer(10 * time.Minute) // 执行的最大时间
Loop:
for {
select {
case ipString := <-this.ipChan:
// serverId@ip@bytes@isAttack
pieces := strings.Split(ipString, "@")
var pieces = strings.Split(ipString, "@")
if len(pieces) < 4 {
continue
}
serverId := pieces[0]
ip := pieces[1]
var serverId = pieces[0]
var ip = pieces[1]
if iplibrary.SharedLibrary != nil {
result, err := iplibrary.SharedLibrary.Lookup(ip)
@@ -216,12 +216,12 @@ Loop:
}
}
case userAgentString := <-this.userAgentChan:
atIndex := strings.Index(userAgentString, "@")
var atIndex = strings.Index(userAgentString, "@")
if atIndex < 0 {
continue
}
serverId := userAgentString[:atIndex]
userAgent := userAgentString[atIndex+1:]
var serverId = userAgentString[:atIndex]
var userAgent = userAgentString[atIndex+1:]
var result = SharedUserAgentParser.Parse(userAgent)
var osInfo = result.OS
@@ -264,12 +264,12 @@ func (this *HTTPRequestStatManager) Upload() error {
}
// 月份相关
pbCities := []*pb.UploadServerHTTPRequestStatRequest_RegionCity{}
pbProviders := []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{}
pbSystems := []*pb.UploadServerHTTPRequestStatRequest_System{}
pbBrowsers := []*pb.UploadServerHTTPRequestStatRequest_Browser{}
var pbCities = []*pb.UploadServerHTTPRequestStatRequest_RegionCity{}
var pbProviders = []*pb.UploadServerHTTPRequestStatRequest_RegionProvider{}
var pbSystems = []*pb.UploadServerHTTPRequestStatRequest_System{}
var pbBrowsers = []*pb.UploadServerHTTPRequestStatRequest_Browser{}
for k, stat := range this.cityMap {
pieces := strings.SplitN(k, "@", 4)
var pieces = strings.SplitN(k, "@", 4)
pbCities = append(pbCities, &pb.UploadServerHTTPRequestStatRequest_RegionCity{
ServerId: types.Int64(pieces[0]),
CountryName: pieces[1],
@@ -282,7 +282,7 @@ func (this *HTTPRequestStatManager) Upload() error {
})
}
for k, count := range this.providerMap {
pieces := strings.SplitN(k, "@", 2)
var pieces = strings.SplitN(k, "@", 2)
pbProviders = append(pbProviders, &pb.UploadServerHTTPRequestStatRequest_RegionProvider{
ServerId: types.Int64(pieces[0]),
Name: pieces[1],
@@ -290,7 +290,7 @@ func (this *HTTPRequestStatManager) Upload() error {
})
}
for k, count := range this.systemMap {
pieces := strings.SplitN(k, "@", 3)
var pieces = strings.SplitN(k, "@", 3)
pbSystems = append(pbSystems, &pb.UploadServerHTTPRequestStatRequest_System{
ServerId: types.Int64(pieces[0]),
Name: pieces[1],
@@ -299,7 +299,7 @@ func (this *HTTPRequestStatManager) Upload() error {
})
}
for k, count := range this.browserMap {
pieces := strings.SplitN(k, "@", 3)
var pieces = strings.SplitN(k, "@", 3)
pbBrowsers = append(pbBrowsers, &pb.UploadServerHTTPRequestStatRequest_Browser{
ServerId: types.Int64(pieces[0]),
Name: pieces[1],
@@ -309,9 +309,9 @@ func (this *HTTPRequestStatManager) Upload() error {
}
// 防火墙相关
pbFirewallRuleGroups := []*pb.UploadServerHTTPRequestStatRequest_HTTPFirewallRuleGroup{}
var pbFirewallRuleGroups = []*pb.UploadServerHTTPRequestStatRequest_HTTPFirewallRuleGroup{}
for k, count := range this.dailyFirewallRuleGroupMap {
pieces := strings.SplitN(k, "@", 3)
var pieces = strings.SplitN(k, "@", 3)
pbFirewallRuleGroups = append(pbFirewallRuleGroups, &pb.UploadServerHTTPRequestStatRequest_HTTPFirewallRuleGroup{
ServerId: types.Int64(pieces[0]),
HttpFirewallRuleGroupId: types.Int64(pieces[1]),

View File

@@ -80,7 +80,7 @@ func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig)
remotelogs.Println("TRAFFIC_STAT_MANAGER", "quit")
ticker.Stop()
})
remotelogs.Println("TRAFFIC_STA_MANAGER", "start ...")
remotelogs.Println("TRAFFIC_STAT_MANAGER", "start ...")
for range ticker.C {
err := this.Upload()
if err != nil {
@@ -95,6 +95,10 @@ func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig)
// Add 添加流量
func (this *TrafficStatManager) Add(serverId int64, domain string, bytes int64, cachedBytes int64, countRequests int64, countCachedRequests int64, countAttacks int64, attackBytes int64, checkingTrafficLimit bool, planId int64) {
if serverId == 0 {
return
}
if bytes == 0 && countRequests == 0 {
return
}
@@ -139,7 +143,7 @@ func (this *TrafficStatManager) Add(serverId int64, domain string, bytes int64,
// Upload 上传流量
func (this *TrafficStatManager) Upload() error {
config := this.configFunc()
var config = this.configFunc()
if config == nil {
return nil
}
@@ -150,8 +154,8 @@ func (this *TrafficStatManager) Upload() error {
}
this.locker.Lock()
itemMap := this.itemMap
domainMap := this.domainsMap
var itemMap = this.itemMap
var domainMap = this.domainsMap
this.itemMap = map[string]*TrafficItem{}
this.domainsMap = map[string]*TrafficItem{}
this.locker.Unlock()

View File

@@ -91,7 +91,7 @@ func (this *Cache) Write(key string, value interface{}, expiredAt int64) (ok boo
})
}
func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64) int64 {
func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64, extend bool) int64 {
if this.isDestroyed {
return 0
}
@@ -107,7 +107,7 @@ func (this *Cache) IncreaseInt64(key string, delta int64, expiredAt int64) int64
}
uint64Key := HashKey([]byte(key))
pieceIndex := uint64Key % this.countPieces
return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiredAt)
return this.pieces[pieceIndex].IncreaseInt64(uint64Key, delta, expiredAt, extend)
}
func (this *Cache) Read(key string) (item *Item) {

View File

@@ -65,14 +65,14 @@ func TestCache_IncreaseInt64(t *testing.T) {
var unixTime = time.Now().Unix()
{
cache.IncreaseInt64("a", 1, unixTime+3600)
cache.IncreaseInt64("a", 1, unixTime+3600, false)
var item = cache.Read("a")
t.Log(item)
a.IsTrue(item.Value == int64(1))
a.IsTrue(item.expiredAt == unixTime+3600)
}
{
cache.IncreaseInt64("a", 1, unixTime+3600+1)
cache.IncreaseInt64("a", 1, unixTime+3600+1, true)
var item = cache.Read("a")
t.Log(item)
a.IsTrue(item.Value == int64(2))
@@ -83,7 +83,7 @@ func TestCache_IncreaseInt64(t *testing.T) {
t.Log(cache.Read("b"))
}
{
cache.IncreaseInt64("b", 1, time.Now().Unix()+3600+3)
cache.IncreaseInt64("b", 1, time.Now().Unix()+3600+3, false)
t.Log(cache.Read("b"))
}
}

View File

@@ -39,13 +39,15 @@ func (this *Piece) Add(key uint64, item *Item) (ok bool) {
return true
}
func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64) (result int64) {
func (this *Piece) IncreaseInt64(key uint64, delta int64, expiredAt int64, extend bool) (result int64) {
this.locker.Lock()
item, ok := this.m[key]
if ok && item.expiredAt > time.Now().Unix() {
result = types.Int64(item.Value) + delta
item.Value = result
item.expiredAt = expiredAt
if extend {
item.expiredAt = expiredAt
}
this.expiresList.Add(key, expiredAt)
} else {
if len(this.m) < this.maxItems {

Some files were not shown because too many files have changed in this diff Show More