Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7dde0deb25 | ||
|
|
b4647b1baa | ||
|
|
952e3ca572 | ||
|
|
238973a5e2 | ||
|
|
9b6ab2fa8b | ||
|
|
9591004b70 | ||
|
|
00cd86a8b3 | ||
|
|
2a1cc63989 | ||
|
|
8177768cf6 | ||
|
|
14d156d42d | ||
|
|
63992bb2a0 | ||
|
|
d02f9f9a0e | ||
|
|
91fab59a18 | ||
|
|
76c82b431a | ||
|
|
2f6414fc55 | ||
|
|
e23f4aaee2 | ||
|
|
443660ac38 | ||
|
|
488430bbef | ||
|
|
344de90bff | ||
|
|
2f02827cb7 | ||
|
|
03e774cc44 | ||
|
|
ff2826ab47 | ||
|
|
ecaa45db34 | ||
|
|
b6cc826a54 | ||
|
|
b8d7e3f5b4 | ||
|
|
390be7f6c6 | ||
|
|
ac4e240912 | ||
|
|
be7267211b | ||
|
|
88fa75acb5 | ||
|
|
d62fccf0a4 | ||
|
|
258ffef0c2 | ||
|
|
a41f834192 | ||
|
|
00500cb6a3 | ||
|
|
32a3400138 | ||
|
|
5ae4ef665e | ||
|
|
336db828ad | ||
|
|
a1212804bb | ||
|
|
763ab4ac98 | ||
|
|
4ec6ae4301 | ||
|
|
4f292c5003 | ||
|
|
a00325f41a | ||
|
|
fc4490b782 | ||
|
|
7f3f7e21b8 |
@@ -6,6 +6,9 @@ function build() {
|
||||
VERSION=$(lookup-version $ROOT/../internal/const/const.go)
|
||||
DIST=$ROOT/"../dist/${NAME}"
|
||||
MUSL_DIR="/usr/local/opt/musl-cross/bin"
|
||||
GCC_X86_64_DIR="/usr/local/Cellar/x86_64-unknown-linux-gnu/10.3.0/bin"
|
||||
GCC_ARM64_DIR="/usr/local/Cellar/aarch64-unknown-linux-gnu/10.3.0/bin"
|
||||
|
||||
OS=${1}
|
||||
ARCH=${2}
|
||||
TAG=${3}
|
||||
@@ -56,19 +59,39 @@ function build() {
|
||||
|
||||
CC_PATH=""
|
||||
CXX_PATH=""
|
||||
BUILD_TAG=$TAG
|
||||
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
|
||||
# /usr/local/opt/musl-cross/bin/
|
||||
if [ "${ARCH}" == "amd64" ]; then
|
||||
CC_PATH="x86_64-linux-musl-gcc"
|
||||
CXX_PATH="x86_64-linux-musl-g++"
|
||||
# build with script support
|
||||
if [ -d $GCC_X86_64_DIR ]; then
|
||||
MUSL_DIR=$GCC_X86_64_DIR
|
||||
CC_PATH="x86_64-unknown-linux-gnu-gcc"
|
||||
CXX_PATH="x86_64-unknown-linux-gnu-g++"
|
||||
if [ "$TAG" = "plus" ]; then
|
||||
BUILD_TAG="plus,script"
|
||||
fi
|
||||
else
|
||||
CC_PATH="x86_64-linux-musl-gcc"
|
||||
CXX_PATH="x86_64-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
if [ "${ARCH}" == "386" ]; then
|
||||
CC_PATH="i486-linux-musl-gcc"
|
||||
CXX_PATH="i486-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "arm64" ]; then
|
||||
CC_PATH="aarch64-linux-musl-gcc"
|
||||
CXX_PATH="aarch64-linux-musl-g++"
|
||||
# build with script support
|
||||
if [ -d $GCC_ARM64_DIR ]; then
|
||||
MUSL_DIR=$GCC_ARM64_DIR
|
||||
CC_PATH="aarch64-unknown-linux-gnu-gcc"
|
||||
CXX_PATH="aarch64-unknown-linux-gnu-g++"
|
||||
if [ "$TAG" = "plus" ]; then
|
||||
BUILD_TAG="plus,script"
|
||||
fi
|
||||
else
|
||||
CC_PATH="aarch64-linux-musl-gcc"
|
||||
CXX_PATH="aarch64-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
if [ "${ARCH}" == "arm" ]; then
|
||||
CC_PATH="arm-linux-musleabi-gcc"
|
||||
@@ -84,7 +107,7 @@ function build() {
|
||||
fi
|
||||
fi
|
||||
if [ ! -z $CC_PATH ]; then
|
||||
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
|
||||
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $BUILD_TAG -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
|
||||
else
|
||||
env GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
|
||||
fi
|
||||
|
||||
@@ -8,7 +8,10 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/nodes"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/iwind/gosock/pkg/gosock"
|
||||
"net"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
@@ -117,6 +120,122 @@ func main() {
|
||||
}
|
||||
}
|
||||
})
|
||||
app.On("gc", func() {
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
_, err := sock.Send(&gosock.Command{Code: "gc"})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
} else {
|
||||
fmt.Println("ok")
|
||||
}
|
||||
})
|
||||
app.On("ip.drop", func() {
|
||||
var args = os.Args[2:]
|
||||
if len(args) == 0 {
|
||||
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS]")
|
||||
return
|
||||
}
|
||||
var ip = args[0]
|
||||
if len(net.ParseIP(ip)) == 0 {
|
||||
fmt.Println("IP '" + ip + "' is invalid")
|
||||
return
|
||||
}
|
||||
var timeoutSeconds = 0
|
||||
var options = app.ParseOptions(args[1:])
|
||||
timeout, ok := options["timeout"]
|
||||
if ok {
|
||||
timeoutSeconds = types.Int(timeout[0])
|
||||
}
|
||||
|
||||
fmt.Println("drop ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{
|
||||
Code: "dropIP",
|
||||
Params: map[string]interface{}{
|
||||
"ip": ip,
|
||||
"timeoutSeconds": timeoutSeconds,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
} else {
|
||||
var errString = maps.NewMap(reply.Params).GetString("error")
|
||||
if len(errString) > 0 {
|
||||
fmt.Println("[ERROR]" + errString)
|
||||
} else {
|
||||
fmt.Println("ok")
|
||||
}
|
||||
}
|
||||
})
|
||||
app.On("ip.reject", func() {
|
||||
var args = os.Args[2:]
|
||||
if len(args) == 0 {
|
||||
fmt.Println("Usage: edge-node ip.reject IP [--timeout=SECONDS]")
|
||||
return
|
||||
}
|
||||
var ip = args[0]
|
||||
if len(net.ParseIP(ip)) == 0 {
|
||||
fmt.Println("IP '" + ip + "' is invalid")
|
||||
return
|
||||
}
|
||||
var timeoutSeconds = 0
|
||||
var options = app.ParseOptions(args[1:])
|
||||
timeout, ok := options["timeout"]
|
||||
if ok {
|
||||
timeoutSeconds = types.Int(timeout[0])
|
||||
}
|
||||
|
||||
fmt.Println("reject ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
|
||||
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{
|
||||
Code: "rejectIP",
|
||||
Params: map[string]interface{}{
|
||||
"ip": ip,
|
||||
"timeoutSeconds": timeoutSeconds,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
} else {
|
||||
var errString = maps.NewMap(reply.Params).GetString("error")
|
||||
if len(errString) > 0 {
|
||||
fmt.Println("[ERROR]" + errString)
|
||||
} else {
|
||||
fmt.Println("ok")
|
||||
}
|
||||
}
|
||||
})
|
||||
app.On("ip.remove", func() {
|
||||
var args = os.Args[2:]
|
||||
if len(args) == 0 {
|
||||
fmt.Println("Usage: edge-node ip.remove IP")
|
||||
return
|
||||
}
|
||||
var ip = args[0]
|
||||
if len(net.ParseIP(ip)) == 0 {
|
||||
fmt.Println("IP '" + ip + "' is invalid")
|
||||
return
|
||||
}
|
||||
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{
|
||||
Code: "removeIP",
|
||||
Params: map[string]interface{}{
|
||||
"ip": ip,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
} else {
|
||||
var errString = maps.NewMap(reply.Params).GetString("error")
|
||||
if len(errString) > 0 {
|
||||
fmt.Println("[ERROR]" + errString)
|
||||
} else {
|
||||
fmt.Println("ok")
|
||||
}
|
||||
}
|
||||
})
|
||||
app.Run(func() {
|
||||
node := nodes.NewNode()
|
||||
node.Start()
|
||||
|
||||
13
go.mod
13
go.mod
@@ -2,7 +2,9 @@ module github.com/TeaOSLab/EdgeNode
|
||||
|
||||
go 1.15
|
||||
|
||||
replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
|
||||
replace (
|
||||
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
|
||||
@@ -11,25 +13,28 @@ require (
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/chai2010/webp v1.1.0 // indirect
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
|
||||
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49
|
||||
github.com/fsnotify/fsnotify v1.5.1 // indirect
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
|
||||
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.9
|
||||
github.com/miekg/dns v1.1.43
|
||||
github.com/mssola/user_agent v0.5.3
|
||||
github.com/pires/go-proxyproto v0.6.1
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible
|
||||
github.com/tklauser/go-sysconf v0.3.6 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9
|
||||
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
|
||||
golang.org/x/text v0.3.7
|
||||
golang.org/x/tools v0.1.3 // indirect
|
||||
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
|
||||
google.golang.org/grpc v1.43.0
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
|
||||
)
|
||||
|
||||
57
go.sum
57
go.sum
@@ -6,8 +6,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
|
||||
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
|
||||
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
@@ -36,28 +34,22 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
|
||||
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49 h1:CtSi0QlA2Hy+nOh8JAZoiEBLW5pliAiKJ3l1Iq1472I=
|
||||
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
|
||||
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
|
||||
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
|
||||
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
|
||||
@@ -89,7 +81,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060 h1:qdLtK4PDXxk2vMKkTWl5Fl9xqYuRCukzWAgJbLHdfOo=
|
||||
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
|
||||
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24 h1:1cGulkD2SNJJRok5OKwyhP/Ddm+PgSWKOupn0cR36/A=
|
||||
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
|
||||
@@ -100,8 +91,6 @@ github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@@ -111,22 +100,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
|
||||
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
|
||||
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
|
||||
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo=
|
||||
github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
|
||||
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
|
||||
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
@@ -145,8 +124,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
|
||||
github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
@@ -154,14 +131,15 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
|
||||
github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
|
||||
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
|
||||
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
@@ -171,18 +149,15 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -196,8 +171,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 h1:kmreh1vGI63l2FxOAYS3Yv6ATsi7lSTuwNSVbGfJV9I=
|
||||
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@@ -214,6 +187,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -226,16 +200,16 @@ golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d h1:1oIt9o40TWWI9FUaveVpUvBe13FNqBNVXy3ue2fcfkw=
|
||||
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
@@ -244,11 +218,7 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
@@ -260,8 +230,6 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
|
||||
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
|
||||
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0=
|
||||
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
@@ -272,8 +240,6 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
|
||||
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
|
||||
google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
|
||||
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
|
||||
@@ -287,7 +253,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
@@ -245,3 +246,19 @@ func (this *AppCmd) getPID() int {
|
||||
}
|
||||
return maps.NewMap(reply.Params).GetInt("pid")
|
||||
}
|
||||
|
||||
// ParseOptions 分析参数中的选项
|
||||
func (this *AppCmd) ParseOptions(args []string) map[string][]string {
|
||||
var result = map[string][]string{}
|
||||
for _, arg := range args {
|
||||
var pieces = strings.SplitN(arg, "=", 2)
|
||||
var key = strings.TrimLeft(pieces[0], "- ")
|
||||
key = strings.TrimSpace(key)
|
||||
var value = ""
|
||||
if len(pieces) == 2 {
|
||||
value = strings.TrimSpace(pieces[1])
|
||||
}
|
||||
result[key] = append(result[key], value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.memoryCache.Write(hash, 1, item.ExpiredAt)
|
||||
atomic.AddInt64(&this.total, 1)
|
||||
|
||||
if this.onAdd != nil {
|
||||
@@ -258,9 +259,10 @@ func (this *FileList) CleanPrefix(prefix string) error {
|
||||
}()
|
||||
|
||||
var count = int64(10000)
|
||||
var staleLife = 600 // TODO 需要可以设置
|
||||
var staleLife = 600 // TODO 需要可以设置
|
||||
var unixTime = utils.UnixTime() // 只删除当前的,不删除新的
|
||||
for {
|
||||
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, utils.UnixTime()+int64(staleLife), utils.UnixTime(), prefix)
|
||||
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -558,9 +560,7 @@ ON "` + this.itemsTableName + `" (
|
||||
// v2 => v3
|
||||
remotelogs.Println("CACHE", "transferring old data from v2 to v3 ...")
|
||||
result, err := db.Exec(`INSERT INTO "` + this.itemsTableName + `" ("id", "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "createdAt", "host", "serverId", "staleAt") SELECT "id", "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "createdAt", "host", "serverId", "expiredAt"+600 FROM cacheItems_v2`)
|
||||
if err != nil {
|
||||
remotelogs.Println("CACHE", "transfer old data from v2 to v3 failed: "+err.Error())
|
||||
} else {
|
||||
if err == nil {
|
||||
count, _ := result.RowsAffected()
|
||||
remotelogs.Println("CACHE", "transfer old data from v2 to v3 finished, "+types.String(count)+" rows transferred")
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
|
||||
for _, policy := range this.policyMap {
|
||||
storage, ok := this.storageMap[policy.Id]
|
||||
if !ok {
|
||||
storage := this.NewStorageWithPolicy(policy)
|
||||
storage = this.NewStorageWithPolicy(policy)
|
||||
if storage == nil {
|
||||
remotelogs.Error("CACHE", "can not find storage type '"+policy.Type+"'")
|
||||
continue
|
||||
@@ -106,7 +106,7 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
|
||||
delete(this.storageMap, policy.Id)
|
||||
|
||||
// 启动新的
|
||||
storage := this.NewStorageWithPolicy(policy)
|
||||
storage = this.NewStorageWithPolicy(policy)
|
||||
if storage == nil {
|
||||
remotelogs.Error("CACHE", "can not find storage type '"+policy.Type+"'")
|
||||
continue
|
||||
|
||||
33
internal/caches/open_file.go
Normal file
33
internal/caches/open_file.go
Normal file
@@ -0,0 +1,33 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type OpenFile struct {
|
||||
fp *os.File
|
||||
meta []byte
|
||||
header []byte
|
||||
version int64
|
||||
}
|
||||
|
||||
func NewOpenFile(fp *os.File, meta []byte, header []byte) *OpenFile {
|
||||
return &OpenFile{
|
||||
fp: fp,
|
||||
meta: meta,
|
||||
header: header,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *OpenFile) SeekStart() error {
|
||||
_, err := this.fp.Seek(0, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *OpenFile) Close() error {
|
||||
this.meta = nil
|
||||
return this.fp.Close()
|
||||
}
|
||||
161
internal/caches/open_file_cache.go
Normal file
161
internal/caches/open_file_cache.go
Normal file
@@ -0,0 +1,161 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type OpenFileCache struct {
|
||||
poolMap map[string]*OpenFilePool // file path => Pool
|
||||
poolList *linkedlist.List
|
||||
watcher *fsnotify.Watcher
|
||||
|
||||
locker sync.Mutex
|
||||
|
||||
maxSize int
|
||||
count int
|
||||
}
|
||||
|
||||
func NewOpenFileCache(maxSize int) (*OpenFileCache, error) {
|
||||
if maxSize <= 0 {
|
||||
maxSize = 16384
|
||||
}
|
||||
|
||||
var cache = &OpenFileCache{
|
||||
maxSize: maxSize,
|
||||
poolMap: map[string]*OpenFilePool{},
|
||||
poolList: linkedlist.NewList(),
|
||||
}
|
||||
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cache.watcher = watcher
|
||||
|
||||
goman.New(func() {
|
||||
for event := range watcher.Events {
|
||||
if event.Op&fsnotify.Chmod != fsnotify.Chmod {
|
||||
cache.Close(event.Name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) Get(filename string) *OpenFile {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
pool, ok := this.poolMap[filename]
|
||||
if ok {
|
||||
file, consumed := pool.Get()
|
||||
if consumed {
|
||||
this.count--
|
||||
}
|
||||
return file
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) Put(filename string, file *OpenFile) {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
pool, ok := this.poolMap[filename]
|
||||
var success bool
|
||||
if ok {
|
||||
success = pool.Put(file)
|
||||
} else {
|
||||
_ = this.watcher.Add(filename)
|
||||
pool = NewOpenFilePool(filename)
|
||||
this.poolMap[filename] = pool
|
||||
success = pool.Put(file)
|
||||
}
|
||||
this.poolList.Push(pool.linkItem)
|
||||
|
||||
// 检查长度
|
||||
if success {
|
||||
this.count++
|
||||
|
||||
// 如果超过当前容量,则关闭最早的
|
||||
if this.count > this.maxSize {
|
||||
var delta = this.maxSize / 100 // 清理1%
|
||||
if delta == 0 {
|
||||
delta = 1
|
||||
}
|
||||
for i := 0; i < delta; i++ {
|
||||
var head = this.poolList.Head()
|
||||
if head == nil {
|
||||
break
|
||||
}
|
||||
|
||||
var headPool = head.Value.(*OpenFilePool)
|
||||
headFile, consumed := headPool.Get()
|
||||
if consumed {
|
||||
this.count--
|
||||
if headFile != nil {
|
||||
_ = headFile.Close()
|
||||
}
|
||||
}
|
||||
|
||||
if headPool.Len() == 0 {
|
||||
delete(this.poolMap, headPool.filename)
|
||||
this.poolList.Remove(head)
|
||||
_ = this.watcher.Remove(headPool.filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) Close(filename string) {
|
||||
this.locker.Lock()
|
||||
|
||||
pool, ok := this.poolMap[filename]
|
||||
if ok {
|
||||
delete(this.poolMap, filename)
|
||||
this.poolList.Remove(pool.linkItem)
|
||||
_ = this.watcher.Remove(filename)
|
||||
this.count -= pool.Len()
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
|
||||
// 在locker之外,提升性能
|
||||
if ok {
|
||||
pool.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) CloseAll() {
|
||||
this.locker.Lock()
|
||||
for _, pool := range this.poolMap {
|
||||
pool.Close()
|
||||
}
|
||||
this.poolMap = map[string]*OpenFilePool{}
|
||||
this.poolList.Reset()
|
||||
_ = this.watcher.Close()
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *OpenFileCache) Debug() {
|
||||
var ticker = time.NewTicker(5 * time.Second)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
logs.Println("==== " + types.String(this.count) + " ====")
|
||||
this.poolList.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
logs.Println(filepath.Base(item.Value.(*OpenFilePool).Filename()), item.Value.(*OpenFilePool).Len())
|
||||
return true
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
76
internal/caches/open_file_pool.go
Normal file
76
internal/caches/open_file_pool.go
Normal file
@@ -0,0 +1,76 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
)
|
||||
|
||||
type OpenFilePool struct {
|
||||
c chan *OpenFile
|
||||
linkItem *linkedlist.Item
|
||||
filename string
|
||||
version int64
|
||||
}
|
||||
|
||||
func NewOpenFilePool(filename string) *OpenFilePool {
|
||||
var pool = &OpenFilePool{
|
||||
filename: filename,
|
||||
c: make(chan *OpenFile, 1024),
|
||||
version: utils.UnixTimeMilli(),
|
||||
}
|
||||
pool.linkItem = linkedlist.NewItem(pool)
|
||||
return pool
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) Filename() string {
|
||||
return this.filename
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) Get() (*OpenFile, bool) {
|
||||
select {
|
||||
case file := <-this.c:
|
||||
err := file.SeekStart()
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, true
|
||||
}
|
||||
file.version = this.version
|
||||
|
||||
return file, true
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) Put(file *OpenFile) bool {
|
||||
if file.version > 0 && file.version != this.version {
|
||||
_ = file.Close()
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case this.c <- file:
|
||||
return true
|
||||
default:
|
||||
// 多余的直接关闭
|
||||
_ = file.Close()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) Len() int {
|
||||
return len(this.c)
|
||||
}
|
||||
|
||||
func (this *OpenFilePool) Close() {
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case file := <-this.c:
|
||||
_ = file.Close()
|
||||
default:
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
}
|
||||
17
internal/caches/open_file_pool_test.go
Normal file
17
internal/caches/open_file_pool_test.go
Normal file
@@ -0,0 +1,17 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOpenFilePool_Get(t *testing.T) {
|
||||
var pool = caches.NewOpenFilePool("a")
|
||||
t.Log(pool.Filename())
|
||||
t.Log(pool.Get())
|
||||
t.Log(pool.Put(caches.NewOpenFile(nil, nil)))
|
||||
t.Log(pool.Get())
|
||||
t.Log(pool.Get())
|
||||
}
|
||||
@@ -11,6 +11,12 @@ import (
|
||||
type FileReader struct {
|
||||
fp *os.File
|
||||
|
||||
openFile *OpenFile
|
||||
openFileCache *OpenFileCache
|
||||
|
||||
meta []byte
|
||||
header []byte
|
||||
|
||||
expiresAt int64
|
||||
status int
|
||||
headerOffset int64
|
||||
@@ -27,6 +33,11 @@ func NewFileReader(fp *os.File) *FileReader {
|
||||
}
|
||||
|
||||
func (this *FileReader) Init() error {
|
||||
if this.openFile != nil {
|
||||
this.meta = this.openFile.meta
|
||||
this.header = this.openFile.header
|
||||
}
|
||||
|
||||
isOk := false
|
||||
|
||||
defer func() {
|
||||
@@ -35,13 +46,17 @@ func (this *FileReader) Init() error {
|
||||
}
|
||||
}()
|
||||
|
||||
var buf = make([]byte, SizeMeta)
|
||||
ok, err := this.readToBuff(this.fp, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
var buf = this.meta
|
||||
if len(buf) == 0 {
|
||||
buf = make([]byte, SizeMeta)
|
||||
ok, err := this.readToBuff(this.fp, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
this.meta = buf
|
||||
}
|
||||
|
||||
this.expiresAt = int64(binary.BigEndian.Uint32(buf[:SizeExpiresAt]))
|
||||
@@ -72,6 +87,21 @@ func (this *FileReader) Init() error {
|
||||
this.bodySize = int64(bodySize)
|
||||
this.bodyOffset = this.headerOffset + int64(headerSize)
|
||||
|
||||
// read header
|
||||
if this.openFileCache != nil && len(this.header) == 0 {
|
||||
if headerSize > 0 && headerSize <= 512 {
|
||||
this.header = make([]byte, headerSize)
|
||||
_, err := this.fp.Seek(this.headerOffset, io.SeekStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = this.readToBuff(this.fp, this.header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isOk = true
|
||||
|
||||
return nil
|
||||
@@ -106,6 +136,22 @@ func (this *FileReader) BodySize() int64 {
|
||||
}
|
||||
|
||||
func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
|
||||
// 使用缓存
|
||||
if len(this.header) > 0 && len(buf) >= len(this.header) {
|
||||
copy(buf, this.header)
|
||||
_, err := callback(len(this.header))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 移动到Body位置
|
||||
_, err = this.fp.Seek(this.bodyOffset, io.SeekStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
isOk := false
|
||||
|
||||
defer func() {
|
||||
@@ -323,6 +369,14 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
|
||||
}
|
||||
|
||||
func (this *FileReader) Close() error {
|
||||
if this.openFileCache != nil {
|
||||
if this.openFile != nil {
|
||||
this.openFileCache.Put(this.fp.Name(), this.openFile)
|
||||
} else {
|
||||
this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, this.meta, this.header))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return this.fp.Close()
|
||||
}
|
||||
|
||||
|
||||
@@ -14,12 +14,12 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"golang.org/x/text/language"
|
||||
"golang.org/x/text/message"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -67,6 +67,8 @@ type FileStorage struct {
|
||||
hotMapLocker sync.Mutex
|
||||
lastHotSize int
|
||||
hotTicker *utils.Ticker
|
||||
|
||||
openFileCache *OpenFileCache
|
||||
}
|
||||
|
||||
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
|
||||
@@ -100,13 +102,13 @@ func (this *FileStorage) Init() error {
|
||||
return err
|
||||
}
|
||||
this.cacheConfig = cacheConfig
|
||||
cacheDir := cacheConfig.Dir
|
||||
|
||||
if !filepath.IsAbs(this.cacheConfig.Dir) {
|
||||
this.cacheConfig.Dir = Tea.Root + Tea.DS + this.cacheConfig.Dir
|
||||
}
|
||||
|
||||
dir := this.cacheConfig.Dir
|
||||
this.cacheConfig.Dir = filepath.Clean(this.cacheConfig.Dir)
|
||||
var dir = this.cacheConfig.Dir
|
||||
|
||||
if len(dir) == 0 {
|
||||
return errors.New("[CACHE]cache storage dir can not be empty")
|
||||
@@ -159,7 +161,7 @@ func (this *FileStorage) Init() error {
|
||||
} else if size > 1024 {
|
||||
sizeMB = fmt.Sprintf("%.3f K", float64(size)/1024)
|
||||
}
|
||||
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+cacheDir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
|
||||
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+this.cacheConfig.Dir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
|
||||
}()
|
||||
|
||||
// 初始化list
|
||||
@@ -202,6 +204,15 @@ func (this *FileStorage) Init() error {
|
||||
}
|
||||
}
|
||||
|
||||
// open file cache
|
||||
if this.cacheConfig.OpenFileCache != nil && this.cacheConfig.OpenFileCache.IsOn && this.cacheConfig.OpenFileCache.Max > 0 {
|
||||
this.openFileCache, err = NewOpenFileCache(this.cacheConfig.OpenFileCache.Max)
|
||||
logs.Println("start open file cache")
|
||||
if err != nil {
|
||||
remotelogs.Error("CACHE", "open file cache failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -238,12 +249,22 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
|
||||
|
||||
// TODO 尝试使用mmap加快读取速度
|
||||
var isOk = false
|
||||
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
var openFile *OpenFile
|
||||
if this.openFileCache != nil {
|
||||
openFile = this.openFileCache.Get(path)
|
||||
}
|
||||
var fp *os.File
|
||||
var err error
|
||||
if openFile == nil {
|
||||
fp, err = os.OpenFile(path, os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
} else {
|
||||
fp = openFile.fp
|
||||
}
|
||||
defer func() {
|
||||
if !isOk {
|
||||
@@ -252,10 +273,9 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
|
||||
}
|
||||
}()
|
||||
|
||||
reader := NewFileReader(fp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var reader = NewFileReader(fp)
|
||||
reader.openFile = openFile
|
||||
reader.openFileCache = this.openFileCache
|
||||
err = reader.Init()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -623,6 +643,8 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
|
||||
|
||||
// Stop 停止
|
||||
func (this *FileStorage) Stop() {
|
||||
events.Remove(this)
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
@@ -640,6 +662,10 @@ func (this *FileStorage) Stop() {
|
||||
}
|
||||
|
||||
_ = this.list.Close()
|
||||
|
||||
if this.openFileCache != nil {
|
||||
this.openFileCache.CloseAll()
|
||||
}
|
||||
}
|
||||
|
||||
// TotalDiskSize 消耗的磁盘尺寸
|
||||
@@ -702,11 +728,20 @@ func (this *FileStorage) initList() error {
|
||||
}
|
||||
}
|
||||
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("CACHE", "quit clean timer")
|
||||
var ticker = this.purgeTicker
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
|
||||
{
|
||||
var ticker = this.purgeTicker
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
}
|
||||
}
|
||||
{
|
||||
var ticker = this.hotTicker
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
}
|
||||
}
|
||||
})
|
||||
goman.New(func() {
|
||||
@@ -733,98 +768,6 @@ func (this *FileStorage) initList() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 解析文件信息
|
||||
func (this *FileStorage) decodeFile(path string) (*Item, error) {
|
||||
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isAllOk := false
|
||||
defer func() {
|
||||
_ = fp.Close()
|
||||
|
||||
if !isAllOk {
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
|
||||
item := &Item{
|
||||
Type: ItemTypeFile,
|
||||
MetaSize: SizeMeta,
|
||||
}
|
||||
|
||||
bytes4 := make([]byte, 4)
|
||||
|
||||
// 过期时间
|
||||
ok, err := this.readToBuff(fp, bytes4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
item.ExpiredAt = int64(binary.BigEndian.Uint32(bytes4))
|
||||
|
||||
// 是否已过期
|
||||
if item.ExpiredAt < time.Now().Unix() {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
// URL Size
|
||||
_, err = fp.Seek(int64(SizeExpiresAt+SizeStatus), io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ok, err = this.readToBuff(fp, bytes4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
urlSize := binary.BigEndian.Uint32(bytes4)
|
||||
|
||||
// Header Size
|
||||
ok, err = this.readToBuff(fp, bytes4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
item.HeaderSize = int64(binary.BigEndian.Uint32(bytes4))
|
||||
|
||||
// Body Size
|
||||
bytes8 := make([]byte, 8)
|
||||
ok, err = this.readToBuff(fp, bytes8)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
item.BodySize = int64(binary.BigEndian.Uint64(bytes8))
|
||||
|
||||
// URL
|
||||
if urlSize > 0 {
|
||||
data := utils.BytePool1k.Get()
|
||||
result, ok, err := this.readN(fp, data, int(urlSize))
|
||||
utils.BytePool1k.Put(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
item.Key = string(result)
|
||||
}
|
||||
|
||||
isAllOk = true
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// 清理任务
|
||||
func (this *FileStorage) purgeLoop() {
|
||||
// 计算是否应该开启LFU清理
|
||||
@@ -1008,34 +951,6 @@ func (this *FileStorage) hotLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileStorage) readToBuff(fp *os.File, buf []byte) (ok bool, err error) {
|
||||
n, err := fp.Read(buf)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ok = n == len(buf)
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileStorage) readN(fp *os.File, buf []byte, total int) (result []byte, ok bool, err error) {
|
||||
for {
|
||||
n, err := fp.Read(buf)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if n > 0 {
|
||||
if n >= total {
|
||||
result = append(result, buf[:total]...)
|
||||
ok = true
|
||||
return result, ok, nil
|
||||
} else {
|
||||
total -= n
|
||||
result = append(result, buf[:n]...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileStorage) diskCapacityBytes() int64 {
|
||||
c1 := this.policy.CapacityBytes()
|
||||
if SharedManager.MaxDiskCapacity != nil {
|
||||
|
||||
@@ -482,11 +482,7 @@ func TestFileStorage_DecodeFile(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, path := storage.keyPath("my-key")
|
||||
item, err := storage.decodeFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
logs.PrintAsJSON(item, t)
|
||||
t.Log(path)
|
||||
}
|
||||
|
||||
func BenchmarkFileStorage_Read(b *testing.B) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build community
|
||||
// +build community
|
||||
|
||||
package teaconst
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build plus
|
||||
// +build plus
|
||||
|
||||
package teaconst
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
package teaconst
|
||||
|
||||
import "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
|
||||
var (
|
||||
// 流量统计
|
||||
|
||||
@@ -10,4 +12,6 @@ var (
|
||||
|
||||
NodeId int64 = 0
|
||||
NodeIdString = ""
|
||||
|
||||
GlobalProductName = nodeconfigs.DefaultProductName
|
||||
)
|
||||
|
||||
@@ -1,27 +1,71 @@
|
||||
package events
|
||||
|
||||
import "sync"
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
var eventsMap = map[string][]func(){} // event => []callbacks
|
||||
type Callbacks = []func()
|
||||
|
||||
var eventsMap = map[Event]map[interface{}]Callbacks{} // event => map[event key][]callback
|
||||
var locker = sync.Mutex{}
|
||||
|
||||
// 增加事件回调
|
||||
func On(event string, callback func()) {
|
||||
var eventKeyId = 0
|
||||
|
||||
func NewKey() interface{} {
|
||||
locker.Lock()
|
||||
defer locker.Unlock()
|
||||
eventKeyId++
|
||||
return eventKeyId
|
||||
}
|
||||
|
||||
// On 增加事件回调
|
||||
func On(event Event, callback func()) {
|
||||
OnKey(event, nil, callback)
|
||||
}
|
||||
|
||||
// OnKey 使用Key增加事件回调
|
||||
func OnKey(event Event, key interface{}, callback func()) {
|
||||
if key == nil {
|
||||
key = NewKey()
|
||||
}
|
||||
|
||||
locker.Lock()
|
||||
defer locker.Unlock()
|
||||
|
||||
callbacks, _ := eventsMap[event]
|
||||
callbacks = append(callbacks, callback)
|
||||
eventsMap[event] = callbacks
|
||||
m, ok := eventsMap[event]
|
||||
if !ok {
|
||||
m = map[interface{}]Callbacks{}
|
||||
eventsMap[event] = m
|
||||
}
|
||||
m[key] = append(m[key], callback)
|
||||
}
|
||||
|
||||
// 通知事件
|
||||
func Notify(event string) {
|
||||
// Remove 删除事件回调
|
||||
func Remove(key interface{}) {
|
||||
if key == nil {
|
||||
return
|
||||
}
|
||||
|
||||
locker.Lock()
|
||||
callbacks, _ := eventsMap[event]
|
||||
for k, m := range eventsMap {
|
||||
_, ok := m[key]
|
||||
if ok {
|
||||
delete(m, key)
|
||||
eventsMap[k] = m
|
||||
}
|
||||
}
|
||||
locker.Unlock()
|
||||
}
|
||||
|
||||
// Notify 通知事件
|
||||
func Notify(event Event) {
|
||||
locker.Lock()
|
||||
m := eventsMap[event]
|
||||
locker.Unlock()
|
||||
|
||||
for _, callback := range callbacks {
|
||||
callback()
|
||||
for _, callbacks := range m {
|
||||
for _, callback := range callbacks {
|
||||
callback()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,33 @@
|
||||
package events
|
||||
package events_test
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOn(t *testing.T) {
|
||||
On("hello", func() {
|
||||
type User struct {
|
||||
name string
|
||||
}
|
||||
var u = &User{}
|
||||
var u2 = &User{}
|
||||
|
||||
events.On("hello", func() {
|
||||
t.Log("world")
|
||||
})
|
||||
On("hello", func() {
|
||||
events.On("hello", func() {
|
||||
t.Log("world2")
|
||||
})
|
||||
On("hello2", func() {
|
||||
events.OnKey("hello", u, func() {
|
||||
t.Log("world3")
|
||||
})
|
||||
events.OnKey("hello", u, func() {
|
||||
t.Log("world4")
|
||||
})
|
||||
events.Remove(u)
|
||||
events.Remove(u2)
|
||||
events.OnKey("hello2", nil, func() {
|
||||
t.Log("world2")
|
||||
})
|
||||
Notify("hello")
|
||||
events.Notify("hello")
|
||||
}
|
||||
|
||||
42
internal/firewalls/firewall.go
Normal file
42
internal/firewalls/firewall.go
Normal file
@@ -0,0 +1,42 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
)
|
||||
|
||||
var currentFirewall FirewallInterface
|
||||
|
||||
// 初始化
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
var firewall = Firewall()
|
||||
if firewall.Name() == "mock" {
|
||||
remotelogs.Warn("FIREWALL", "'firewalld' on this system should be enabled to block attackers more effectively")
|
||||
} else {
|
||||
remotelogs.Println("FIREWALL", "found local firewall '"+firewall.Name()+"'")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Firewall 查找当前系统中最适合的防火墙
|
||||
func Firewall() FirewallInterface {
|
||||
if currentFirewall != nil {
|
||||
return currentFirewall
|
||||
}
|
||||
|
||||
// firewalld
|
||||
{
|
||||
var firewalld = NewFirewalld()
|
||||
if firewalld.IsReady() {
|
||||
currentFirewall = firewalld
|
||||
return currentFirewall
|
||||
}
|
||||
}
|
||||
|
||||
// 至少返回一个
|
||||
currentFirewall = NewMockFirewall()
|
||||
return currentFirewall
|
||||
}
|
||||
135
internal/firewalls/firewall_firewalld.go
Normal file
135
internal/firewalls/firewall_firewalld.go
Normal file
@@ -0,0 +1,135 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Firewalld struct {
|
||||
isReady bool
|
||||
exe string
|
||||
cmdQueue chan *exec.Cmd
|
||||
}
|
||||
|
||||
func NewFirewalld() *Firewalld {
|
||||
var firewalld = &Firewalld{
|
||||
cmdQueue: make(chan *exec.Cmd, 4096),
|
||||
}
|
||||
|
||||
path, err := exec.LookPath("firewall-cmd")
|
||||
if err == nil && len(path) > 0 {
|
||||
var cmd = exec.Command(path, "-V")
|
||||
err := cmd.Run()
|
||||
if err == nil {
|
||||
firewalld.exe = path
|
||||
firewalld.isReady = true
|
||||
firewalld.init()
|
||||
}
|
||||
}
|
||||
|
||||
return firewalld
|
||||
}
|
||||
|
||||
func (this *Firewalld) init() {
|
||||
goman.New(func() {
|
||||
for cmd := range this.cmdQueue {
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
if strings.HasPrefix(err.Error(), "Warning:") {
|
||||
continue
|
||||
}
|
||||
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Name 名称
|
||||
func (this *Firewalld) Name() string {
|
||||
return "firewalld"
|
||||
}
|
||||
|
||||
func (this *Firewalld) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *Firewalld) AllowPort(port int, protocol string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
|
||||
this.pushCmd(cmd)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) RemovePort(port int, protocol string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
|
||||
this.pushCmd(cmd)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
}
|
||||
var args = []string{"--add-rich-rule=rule family='" + family + "' source address='" + ip + "' reject"}
|
||||
if timeoutSeconds > 0 {
|
||||
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
|
||||
}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
}
|
||||
var args = []string{"--add-rich-rule=rule family='" + family + "' source address='" + ip + "' drop"}
|
||||
if timeoutSeconds > 0 {
|
||||
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
|
||||
}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) RemoveSourceIP(ip string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
}
|
||||
for _, action := range []string{"reject", "drop"} {
|
||||
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
|
||||
select {
|
||||
case this.cmdQueue <- cmd:
|
||||
default:
|
||||
// we discard the command
|
||||
}
|
||||
}
|
||||
27
internal/firewalls/firewall_interface.go
Normal file
27
internal/firewalls/firewall_interface.go
Normal file
@@ -0,0 +1,27 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
// FirewallInterface 防火墙接口
|
||||
type FirewallInterface interface {
|
||||
// Name 名称
|
||||
Name() string
|
||||
|
||||
// IsReady 是否已准备被调用
|
||||
IsReady() bool
|
||||
|
||||
// AllowPort 允许端口
|
||||
AllowPort(port int, protocol string) error
|
||||
|
||||
// RemovePort 删除端口
|
||||
RemovePort(port int, protocol string) error
|
||||
|
||||
// RejectSourceIP 拒绝某个源IP连接
|
||||
RejectSourceIP(ip string, timeoutSeconds int) error
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
DropSourceIP(ip string, timeoutSeconds int) error
|
||||
|
||||
// RemoveSourceIP 删除某个源IP
|
||||
RemoveSourceIP(ip string) error
|
||||
}
|
||||
55
internal/firewalls/firewall_mock.go
Normal file
55
internal/firewalls/firewall_mock.go
Normal file
@@ -0,0 +1,55 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
// MockFirewall 模拟防火墙
|
||||
type MockFirewall struct {
|
||||
}
|
||||
|
||||
func NewMockFirewall() *MockFirewall {
|
||||
return &MockFirewall{}
|
||||
}
|
||||
|
||||
// Name 名称
|
||||
func (this *MockFirewall) Name() string {
|
||||
return "mock"
|
||||
}
|
||||
|
||||
// IsReady 是否已准备被调用
|
||||
func (this *MockFirewall) IsReady() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// AllowPort 允许端口
|
||||
func (this *MockFirewall) AllowPort(port int, protocol string) error {
|
||||
_ = port
|
||||
_ = protocol
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemovePort 删除端口
|
||||
func (this *MockFirewall) RemovePort(port int, protocol string) error {
|
||||
_ = port
|
||||
_ = protocol
|
||||
return nil
|
||||
}
|
||||
|
||||
// RejectSourceIP 拒绝某个源IP连接
|
||||
func (this *MockFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
_ = ip
|
||||
_ = timeoutSeconds
|
||||
return nil
|
||||
}
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
_ = ip
|
||||
_ = timeoutSeconds
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveSourceIP 删除某个源IP
|
||||
func (this *MockFirewall) RemoveSourceIP(ip string) error {
|
||||
_ = ip
|
||||
return nil
|
||||
}
|
||||
@@ -71,7 +71,7 @@ func (this *HTTPAPIAction) runAction(action string, listType IPListType, item *p
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", "GoEdge-Node/"+teaconst.Version)
|
||||
req.Header.Set("User-Agent", teaconst.GlobalProductName+"-Node/"+teaconst.Version)
|
||||
resp, err := httpAPIClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
)
|
||||
|
||||
// AllowIP 检查IP是否被允许访问
|
||||
// 如果一个IP不在任何名单中,则允许访问
|
||||
func AllowIP(ip string, serverId int64) bool {
|
||||
var ipLong = utils.IP2Long(ip)
|
||||
if ipLong == 0 {
|
||||
@@ -40,6 +41,17 @@ func AllowIP(ip string, serverId int64) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsInWhiteList 检查IP是否在白名单中
|
||||
func IsInWhiteList(ip string) bool {
|
||||
var ipLong = utils.IP2Long(ip)
|
||||
if ipLong == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// check white lists
|
||||
return GlobalWhiteIPList.Contains(ipLong)
|
||||
}
|
||||
|
||||
// AllowIPStrings 检查一组IP是否被允许访问
|
||||
func AllowIPStrings(ipStrings []string, serverId int64) bool {
|
||||
if len(ipStrings) == 0 {
|
||||
|
||||
156
internal/iplibrary/manager_city.go
Normal file
156
internal/iplibrary/manager_city.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedCityManager = NewCityManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedCityManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedCityManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// CityManager 中国省份信息管理
|
||||
type CityManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
cityMap map[string]int64 // provinceName_cityName => cityName
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewCityManager() *CityManager {
|
||||
return &CityManager{
|
||||
cacheFile: Tea.Root + "/configs/region_city.json.cache",
|
||||
cityMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("CITY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("City_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("CITY_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
|
||||
this.locker.RLock()
|
||||
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *CityManager) load() error {
|
||||
data, err := ioutil.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.cityMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新城市信息
|
||||
func (this *CityManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionCityRPC().FindAllEnabledRegionCities(rpcClient.Context(), &pb.FindAllEnabledRegionCitiesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, city := range resp.RegionCities {
|
||||
for _, code := range city.Codes {
|
||||
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.cityMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = ioutil.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
14
internal/iplibrary/manager_city_test.go
Normal file
14
internal/iplibrary/manager_city_test.go
Normal file
@@ -0,0 +1,14 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package iplibrary
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewCityManager(t *testing.T) {
|
||||
var manager = NewCityManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup(16, "许昌市"))
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"io/ioutil"
|
||||
@@ -26,16 +25,23 @@ func init() {
|
||||
SharedCountryManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedCountryManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// CountryManager 国家/地区信息管理
|
||||
type CountryManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
countryMap map[string]int64 // countryName => countryId
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewCountryManager() *CountryManager {
|
||||
@@ -59,11 +65,8 @@ func (this *CountryManager) Start() {
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
ticker := utils.NewTicker(1 * time.Hour)
|
||||
events.On(events.EventQuit, func() {
|
||||
ticker.Stop()
|
||||
})
|
||||
for ticker.Next() {
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
@@ -71,6 +74,12 @@ func (this *CountryManager) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
|
||||
this.locker.RLock()
|
||||
countryId, _ = this.countryMap[countryName]
|
||||
@@ -101,6 +110,10 @@ func (this *CountryManager) load() error {
|
||||
|
||||
// 更新国家信息
|
||||
func (this *CountryManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -111,7 +124,7 @@ func (this *CountryManager) loop() error {
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, country := range resp.Countries {
|
||||
for _, country := range resp.RegionCountries {
|
||||
for _, code := range country.Codes {
|
||||
m[code] = country.Id
|
||||
}
|
||||
@@ -132,6 +145,7 @@ func (this *CountryManager) loop() error {
|
||||
|
||||
this.locker.Lock()
|
||||
this.countryMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
@@ -23,10 +23,15 @@ func init() {
|
||||
SharedIPListManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedIPListManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// IPListManager IP名单管理
|
||||
type IPListManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
db *IPListDB
|
||||
|
||||
version int64
|
||||
@@ -52,17 +57,14 @@ func (this *IPListManager) Start() {
|
||||
remotelogs.ErrorObject("IP_LIST_MANAGER", err)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(60 * time.Second)
|
||||
this.ticker = time.NewTicker(60 * time.Second)
|
||||
if Tea.IsTesting() {
|
||||
ticker = time.NewTicker(10 * time.Second)
|
||||
this.ticker = time.NewTicker(10 * time.Second)
|
||||
}
|
||||
events.On(events.EventQuit, func() {
|
||||
ticker.Stop()
|
||||
})
|
||||
countErrors := 0
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
case <-this.ticker.C:
|
||||
case <-IPListUpdateNotify:
|
||||
}
|
||||
err := this.loop()
|
||||
@@ -84,6 +86,12 @@ func (this *IPListManager) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *IPListManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *IPListManager) init() {
|
||||
// 从数据库中当中读取数据
|
||||
db, err := NewIPListDB()
|
||||
@@ -197,7 +205,7 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool)
|
||||
list.Delete(item.Id)
|
||||
|
||||
// 从WAF名单中删除
|
||||
waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId)
|
||||
waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId, shouldExecute)
|
||||
|
||||
// 操作事件
|
||||
if shouldExecute {
|
||||
|
||||
155
internal/iplibrary/manager_provider.go
Normal file
155
internal/iplibrary/manager_provider.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedProviderManager = NewProviderManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedProviderManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedProviderManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// ProviderManager 中国省份信息管理
|
||||
type ProviderManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
providerMap map[string]int64 // name => id
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewProviderManager() *ProviderManager {
|
||||
return &ProviderManager{
|
||||
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
|
||||
providerMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
|
||||
this.locker.RLock()
|
||||
providerId, _ = this.providerMap[providerName]
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *ProviderManager) load() error {
|
||||
data, err := ioutil.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.providerMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新服务商信息
|
||||
func (this *ProviderManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionProviderRPC().FindAllEnabledRegionProviders(rpcClient.Context(), &pb.FindAllEnabledRegionProvidersRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, provider := range resp.RegionProviders {
|
||||
for _, code := range provider.Codes {
|
||||
m[code] = provider.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.providerMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = ioutil.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
15
internal/iplibrary/manager_provider_test.go
Normal file
15
internal/iplibrary/manager_provider_test.go
Normal file
@@ -0,0 +1,15 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package iplibrary
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewProviderManager(t *testing.T) {
|
||||
var manager = NewProviderManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("阿里云"))
|
||||
t.Log(manager.Lookup("阿里云2"))
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"io/ioutil"
|
||||
@@ -30,16 +29,23 @@ func init() {
|
||||
SharedProvinceManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedProvinceManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// ProvinceManager 中国省份信息管理
|
||||
type ProvinceManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
provinceMap map[string]int64 // provinceName => provinceId
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewProvinceManager() *ProvinceManager {
|
||||
@@ -63,11 +69,8 @@ func (this *ProvinceManager) Start() {
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
ticker := utils.NewTicker(1 * time.Hour)
|
||||
events.On(events.EventQuit, func() {
|
||||
ticker.Stop()
|
||||
})
|
||||
for ticker.Next() {
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
@@ -75,6 +78,12 @@ func (this *ProvinceManager) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
|
||||
this.locker.RLock()
|
||||
provinceId, _ = this.provinceMap[provinceName]
|
||||
@@ -103,21 +112,25 @@ func (this *ProvinceManager) load() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新国家信息
|
||||
// 更新省份信息
|
||||
func (this *ProvinceManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionProvinceRPC().FindAllEnabledRegionProvincesWithCountryId(rpcClient.Context(), &pb.FindAllEnabledRegionProvincesWithCountryIdRequest{
|
||||
CountryId: ChinaCountryId,
|
||||
RegionCountryId: ChinaCountryId,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, province := range resp.Provinces {
|
||||
for _, province := range resp.RegionProvinces {
|
||||
for _, code := range province.Codes {
|
||||
m[code] = province.Id
|
||||
}
|
||||
@@ -138,6 +151,7 @@ func (this *ProvinceManager) loop() error {
|
||||
|
||||
this.locker.Lock()
|
||||
this.provinceMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
@@ -15,15 +15,22 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedUpdater = NewUpdater()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
updater := NewUpdater()
|
||||
updater.Start()
|
||||
goman.New(func() {
|
||||
SharedUpdater.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedUpdater.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// Updater IP库更新程序
|
||||
type Updater struct {
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
// NewUpdater 获取新对象
|
||||
@@ -34,15 +41,19 @@ func NewUpdater() *Updater {
|
||||
// Start 开始更新
|
||||
func (this *Updater) Start() {
|
||||
// 这里不需要太频繁检查更新,因为通常不需要更新IP库
|
||||
ticker := time.NewTicker(1 * time.Hour)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("IP_LIBRARY", err)
|
||||
}
|
||||
this.ticker = time.NewTicker(1 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("IP_LIBRARY", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Updater) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// 单次任务
|
||||
|
||||
@@ -41,7 +41,7 @@ func NewAPIStream() *APIStream {
|
||||
}
|
||||
|
||||
func (this *APIStream) Start() {
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
this.isQuiting = true
|
||||
if this.cancelFunc != nil {
|
||||
this.cancelFunc()
|
||||
|
||||
@@ -4,9 +4,16 @@ package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ratelimit"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -17,8 +24,11 @@ type ClientConn struct {
|
||||
once sync.Once
|
||||
globalLimiter *ratelimit.Counter
|
||||
|
||||
isTLS bool
|
||||
hasRead bool
|
||||
isTLS bool
|
||||
hasDeadline bool
|
||||
hasRead bool
|
||||
|
||||
hasResetSYNFlood bool
|
||||
|
||||
BaseClientConn
|
||||
}
|
||||
@@ -38,9 +48,9 @@ func NewClientConn(conn net.Conn, isTLS bool, quickClose bool, globalLimiter *ra
|
||||
|
||||
func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
if this.isTLS {
|
||||
if !this.hasRead {
|
||||
if !this.hasDeadline {
|
||||
_ = this.rawConn.SetReadDeadline(time.Now().Add(time.Duration(nodeconfigs.DefaultTLSHandshakeTimeout) * time.Second)) // TODO 握手超时时间可以设置
|
||||
this.hasRead = true
|
||||
this.hasDeadline = true
|
||||
defer func() {
|
||||
_ = this.rawConn.SetReadDeadline(time.Time{})
|
||||
}()
|
||||
@@ -50,7 +60,26 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
n, err = this.rawConn.Read(b)
|
||||
if n > 0 {
|
||||
atomic.AddUint64(&teaconst.InTrafficBytes, uint64(n))
|
||||
if !this.hasRead {
|
||||
this.hasRead = true
|
||||
}
|
||||
}
|
||||
|
||||
// SYN Flood检测
|
||||
var isHandshakeError = err != nil && os.IsTimeout(err) && !this.hasRead
|
||||
if isHandshakeError {
|
||||
_ = this.SetLinger(0)
|
||||
}
|
||||
var synFloodConfig = sharedNodeConfig.SYNFloodConfig()
|
||||
if synFloodConfig != nil && synFloodConfig.IsOn {
|
||||
if isHandshakeError {
|
||||
this.increaseSYNFlood(synFloodConfig)
|
||||
} else if err == nil && !this.hasResetSYNFlood {
|
||||
this.hasResetSYNFlood = true
|
||||
this.resetSYNFlood()
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -99,3 +128,30 @@ func (this *ClientConn) SetReadDeadline(t time.Time) error {
|
||||
func (this *ClientConn) SetWriteDeadline(t time.Time) error {
|
||||
return this.rawConn.SetWriteDeadline(t)
|
||||
}
|
||||
|
||||
func (this *ClientConn) resetSYNFlood() {
|
||||
ttlcache.SharedCache.Delete("SYN_FLOOD:" + this.RawIP())
|
||||
}
|
||||
|
||||
func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloodConfig) {
|
||||
var ip = this.RawIP()
|
||||
if len(ip) > 0 && !iplibrary.IsInWhiteList(ip) && (!synFloodConfig.IgnoreLocal || !utils.IsLocalIP(ip)) {
|
||||
var timestamp = utils.NextMinuteUnixTime()
|
||||
var result = ttlcache.SharedCache.IncreaseInt64("SYN_FLOOD:"+ip, 1, timestamp)
|
||||
var minAttempts = synFloodConfig.MinAttempts
|
||||
if minAttempts < 5 {
|
||||
minAttempts = 5
|
||||
}
|
||||
if !this.isTLS {
|
||||
// 非TLS,设置为两倍,防止误封
|
||||
minAttempts = 2 * minAttempts
|
||||
}
|
||||
if result >= int64(minAttempts) {
|
||||
var timeout = synFloodConfig.TimeoutSeconds
|
||||
if timeout <= 0 {
|
||||
timeout = 600
|
||||
}
|
||||
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击,当前1分钟"+types.String(result)+"次空连接")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,3 +36,23 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
|
||||
return sharedClientConnLimiter.Add(this.rawConn.RemoteAddr().String(), serverId, remoteAddr, maxConnsPerServer, maxConnsPerIP)
|
||||
}
|
||||
|
||||
// RawIP 原本IP
|
||||
func (this *BaseClientConn) RawIP() string {
|
||||
ip, _, _ := net.SplitHostPort(this.rawConn.RemoteAddr().String())
|
||||
return ip
|
||||
}
|
||||
|
||||
// TCPConn 转换为TCPConn
|
||||
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
|
||||
conn, ok := this.rawConn.(*net.TCPConn)
|
||||
return conn, ok
|
||||
}
|
||||
|
||||
// SetLinger 设置Linger
|
||||
func (this *BaseClientConn) SetLinger(seconds int) error {
|
||||
tcpConn, ok := this.TCPConn()
|
||||
if ok {
|
||||
return tcpConn.SetLinger(seconds)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
// 发送监控流量
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
var ticker = time.NewTicker(1 * time.Minute)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
// 加入到数据队列中
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -108,37 +109,33 @@ Loop:
|
||||
}
|
||||
|
||||
func (this *HTTPAccessLogQueue) toValidUTF8(accessLog *pb.HTTPAccessLog) {
|
||||
accessLog.RemoteUser = this.toValidUTF8string(accessLog.RemoteUser)
|
||||
accessLog.RequestURI = this.toValidUTF8string(accessLog.RequestURI)
|
||||
accessLog.RequestPath = this.toValidUTF8string(accessLog.RequestPath)
|
||||
accessLog.RequestFilename = this.toValidUTF8string(accessLog.RequestFilename)
|
||||
accessLog.RemoteUser = utils.ToValidUTF8string(accessLog.RemoteUser)
|
||||
accessLog.RequestURI = utils.ToValidUTF8string(accessLog.RequestURI)
|
||||
accessLog.RequestPath = utils.ToValidUTF8string(accessLog.RequestPath)
|
||||
accessLog.RequestFilename = utils.ToValidUTF8string(accessLog.RequestFilename)
|
||||
accessLog.RequestBody = bytes.ToValidUTF8(accessLog.RequestBody, []byte{})
|
||||
|
||||
for _, v := range accessLog.SentHeader {
|
||||
for index, s := range v.Values {
|
||||
v.Values[index] = this.toValidUTF8string(s)
|
||||
v.Values[index] = utils.ToValidUTF8string(s)
|
||||
}
|
||||
}
|
||||
|
||||
accessLog.Referer = this.toValidUTF8string(accessLog.Referer)
|
||||
accessLog.UserAgent = this.toValidUTF8string(accessLog.UserAgent)
|
||||
accessLog.Request = this.toValidUTF8string(accessLog.Request)
|
||||
accessLog.ContentType = this.toValidUTF8string(accessLog.ContentType)
|
||||
accessLog.Referer = utils.ToValidUTF8string(accessLog.Referer)
|
||||
accessLog.UserAgent = utils.ToValidUTF8string(accessLog.UserAgent)
|
||||
accessLog.Request = utils.ToValidUTF8string(accessLog.Request)
|
||||
accessLog.ContentType = utils.ToValidUTF8string(accessLog.ContentType)
|
||||
|
||||
for k, c := range accessLog.Cookie {
|
||||
accessLog.Cookie[k] = this.toValidUTF8string(c)
|
||||
accessLog.Cookie[k] = utils.ToValidUTF8string(c)
|
||||
}
|
||||
|
||||
accessLog.Args = this.toValidUTF8string(accessLog.Args)
|
||||
accessLog.QueryString = this.toValidUTF8string(accessLog.QueryString)
|
||||
accessLog.Args = utils.ToValidUTF8string(accessLog.Args)
|
||||
accessLog.QueryString = utils.ToValidUTF8string(accessLog.QueryString)
|
||||
|
||||
for _, v := range accessLog.Header {
|
||||
for index, s := range v.Values {
|
||||
v.Values[index] = this.toValidUTF8string(s)
|
||||
v.Values[index] = utils.ToValidUTF8string(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *HTTPAccessLogQueue) toValidUTF8string(v string) string {
|
||||
return strings.ToValidUTF8(v, "")
|
||||
}
|
||||
|
||||
@@ -84,14 +84,13 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
|
||||
if idleConns <= 0 {
|
||||
idleConns = numberCPU * 8
|
||||
}
|
||||
//logs.Println("[ORIGIN]max connections:", maxConnections)
|
||||
|
||||
// TLS通讯
|
||||
tlsConfig := &tls.Config{
|
||||
var tlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
if origin.Cert != nil {
|
||||
obj := origin.Cert.CertObject()
|
||||
var obj = origin.Cert.CertObject()
|
||||
if obj != nil {
|
||||
tlsConfig.InsecureSkipVerify = false
|
||||
tlsConfig.Certificates = []tls.Certificate{*obj}
|
||||
@@ -101,37 +100,16 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
|
||||
}
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
var transport = &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
// 支持TOA的连接
|
||||
toaConfig := sharedTOAManager.Config()
|
||||
if toaConfig != nil && toaConfig.IsOn {
|
||||
retries := 3
|
||||
for i := 1; i <= retries; i++ {
|
||||
port := int(toaConfig.RandLocalPort())
|
||||
// TODO 思考是否支持X-Real-IP/X-Forwarded-IP
|
||||
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + req.requestRemoteAddr(true))
|
||||
if err != nil {
|
||||
remotelogs.Error("TOA", "add failed: "+err.Error())
|
||||
} else {
|
||||
dialer := net.Dialer{
|
||||
Timeout: connectionTimeout,
|
||||
KeepAlive: 1 * time.Minute,
|
||||
LocalAddr: &net.TCPAddr{
|
||||
Port: port,
|
||||
},
|
||||
}
|
||||
conn, err := dialer.DialContext(ctx, network, originAddr)
|
||||
// TODO 需要在合适的时机删除TOA记录
|
||||
if err == nil || i == retries {
|
||||
return conn, err
|
||||
}
|
||||
}
|
||||
}
|
||||
conn, err := this.handleTOA(req, ctx, network, originAddr, connectionTimeout)
|
||||
if conn != nil || err != nil {
|
||||
return conn, err
|
||||
}
|
||||
|
||||
// 普通的连接
|
||||
conn, err := (&net.Dialer{
|
||||
conn, err = (&net.Dialer{
|
||||
Timeout: connectionTimeout,
|
||||
KeepAlive: 1 * time.Minute,
|
||||
}).DialContext(ctx, network, originAddr)
|
||||
@@ -139,32 +117,10 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if proxyProtocol != nil && proxyProtocol.IsOn && (proxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || proxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
|
||||
var remoteAddr = req.requestRemoteAddr(true)
|
||||
var transportProtocol = proxyproto.TCPv4
|
||||
if strings.Contains(remoteAddr, ":") {
|
||||
transportProtocol = proxyproto.TCPv6
|
||||
}
|
||||
var destAddr = conn.RemoteAddr()
|
||||
var reqConn = req.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if reqConn != nil {
|
||||
destAddr = reqConn.(net.Conn).LocalAddr()
|
||||
}
|
||||
header := proxyproto.Header{
|
||||
Version: byte(proxyProtocol.Version),
|
||||
Command: proxyproto.PROXY,
|
||||
TransportProtocol: transportProtocol,
|
||||
SourceAddr: &net.TCPAddr{
|
||||
IP: net.ParseIP(remoteAddr),
|
||||
Port: req.requestRemotePort(),
|
||||
},
|
||||
DestinationAddr: destAddr,
|
||||
}
|
||||
_, err = header.WriteTo(conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
// 处理PROXY protocol
|
||||
err = this.handlePROXYProtocol(conn, req, proxyProtocol)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
@@ -174,7 +130,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
|
||||
MaxConnsPerHost: maxConnections,
|
||||
IdleConnTimeout: idleTimeout,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
TLSHandshakeTimeout: 3 * time.Second,
|
||||
TLSClientConfig: tlsConfig,
|
||||
Proxy: nil,
|
||||
}
|
||||
@@ -208,3 +164,69 @@ func (this *HTTPClientPool) cleanClients() {
|
||||
this.locker.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// 支持TOA
|
||||
func (this *HTTPClientPool) handleTOA(req *HTTPRequest, ctx context.Context, network string, originAddr string, connectionTimeout time.Duration) (net.Conn, error) {
|
||||
// TODO 每个服务读取自身所属集群的TOA设置
|
||||
toaConfig := sharedTOAManager.Config()
|
||||
if toaConfig != nil && toaConfig.IsOn {
|
||||
retries := 3
|
||||
for i := 1; i <= retries; i++ {
|
||||
port := int(toaConfig.RandLocalPort())
|
||||
// TODO 思考是否支持X-Real-IP/X-Forwarded-IP
|
||||
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + req.requestRemoteAddr(true))
|
||||
if err != nil {
|
||||
remotelogs.Error("TOA", "add failed: "+err.Error())
|
||||
} else {
|
||||
dialer := net.Dialer{
|
||||
Timeout: connectionTimeout,
|
||||
KeepAlive: 1 * time.Minute,
|
||||
LocalAddr: &net.TCPAddr{
|
||||
Port: port,
|
||||
},
|
||||
}
|
||||
conn, err := dialer.DialContext(ctx, network, originAddr)
|
||||
// TODO 需要在合适的时机删除TOA记录
|
||||
if err == nil || i == retries {
|
||||
return conn, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// 支持PROXY Protocol
|
||||
func (this *HTTPClientPool) handlePROXYProtocol(conn net.Conn, req *HTTPRequest, proxyProtocol *serverconfigs.ProxyProtocolConfig) error {
|
||||
if proxyProtocol != nil && proxyProtocol.IsOn && (proxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || proxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
|
||||
var remoteAddr = req.requestRemoteAddr(true)
|
||||
var transportProtocol = proxyproto.TCPv4
|
||||
if strings.Contains(remoteAddr, ":") {
|
||||
transportProtocol = proxyproto.TCPv6
|
||||
}
|
||||
var destAddr = conn.RemoteAddr()
|
||||
var reqConn = req.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if reqConn != nil {
|
||||
destAddr = reqConn.(net.Conn).LocalAddr()
|
||||
}
|
||||
header := proxyproto.Header{
|
||||
Version: byte(proxyProtocol.Version),
|
||||
Command: proxyproto.PROXY,
|
||||
TransportProtocol: transportProtocol,
|
||||
SourceAddr: &net.TCPAddr{
|
||||
IP: net.ParseIP(remoteAddr),
|
||||
Port: req.requestRemotePort(),
|
||||
},
|
||||
DestinationAddr: destAddr,
|
||||
}
|
||||
_, err := header.WriteTo(conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,10 +8,12 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/metrics"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -38,8 +40,8 @@ type HTTPRequest struct {
|
||||
// 外部参数
|
||||
RawReq *http.Request
|
||||
RawWriter http.ResponseWriter
|
||||
Server *serverconfigs.ServerConfig
|
||||
Host string // 请求的Host
|
||||
ReqServer *serverconfigs.ServerConfig
|
||||
ReqHost string // 请求的Host
|
||||
ServerName string // 实际匹配到的Host
|
||||
ServerAddr string // 实际启动的服务器监听地址
|
||||
IsHTTP bool
|
||||
@@ -63,6 +65,7 @@ type HTTPRequest struct {
|
||||
rewriteRule *serverconfigs.HTTPRewriteRule // 匹配到的重写规则
|
||||
rewriteReplace string // 重写规则的目标
|
||||
rewriteIsExternalURL bool // 重写目标是否为外部URL
|
||||
remoteAddr string // 计算后的RemoteAddr
|
||||
|
||||
cacheRef *serverconfigs.HTTPCacheRef // 缓存设置
|
||||
cacheKey string // 缓存使用的Key
|
||||
@@ -83,6 +86,9 @@ type HTTPRequest struct {
|
||||
logAttrs map[string]string
|
||||
|
||||
disableLog bool // 此请求中关闭Log
|
||||
|
||||
// script相关操作
|
||||
isDone bool
|
||||
}
|
||||
|
||||
// 初始化
|
||||
@@ -95,7 +101,7 @@ func (this *HTTPRequest) init() {
|
||||
// this.uri = this.RawReq.URL.RequestURI()
|
||||
// 之所以不使用RequestURI(),是不想让URL中的Path被Encode
|
||||
var urlPath = this.RawReq.URL.Path
|
||||
if this.Server.Web != nil && this.Server.Web.MergeSlashes {
|
||||
if this.ReqServer.Web != nil && this.ReqServer.Web.MergeSlashes {
|
||||
urlPath = utils.CleanPath(urlPath)
|
||||
this.web.MergeSlashes = true
|
||||
}
|
||||
@@ -126,19 +132,26 @@ func (this *HTTPRequest) Do() {
|
||||
this.init()
|
||||
|
||||
// 当前服务的反向代理配置
|
||||
if this.Server.ReverseProxyRef != nil && this.Server.ReverseProxy != nil {
|
||||
this.reverseProxyRef = this.Server.ReverseProxyRef
|
||||
this.reverseProxy = this.Server.ReverseProxy
|
||||
if this.ReqServer.ReverseProxyRef != nil && this.ReqServer.ReverseProxy != nil {
|
||||
this.reverseProxyRef = this.ReqServer.ReverseProxyRef
|
||||
this.reverseProxy = this.ReqServer.ReverseProxy
|
||||
}
|
||||
|
||||
// Web配置
|
||||
err := this.configureWeb(this.Server.Web, true, 0)
|
||||
err := this.configureWeb(this.ReqServer.Web, true, 0)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
|
||||
// 回调事件
|
||||
this.onInit()
|
||||
if this.writer.isFinished {
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
|
||||
// 特殊URL处理
|
||||
if len(this.rawURI) > 1 && this.rawURI[1] == '.' {
|
||||
// ACME
|
||||
@@ -151,14 +164,14 @@ func (this *HTTPRequest) Do() {
|
||||
}
|
||||
|
||||
// 套餐
|
||||
if this.Server.UserPlan != nil && !this.Server.UserPlan.IsAvailable() {
|
||||
if this.ReqServer.UserPlan != nil && !this.ReqServer.UserPlan.IsAvailable() {
|
||||
this.doPlanExpires()
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
|
||||
// 流量限制
|
||||
if this.Server.TrafficLimit != nil && this.Server.TrafficLimit.IsOn && !this.Server.TrafficLimit.IsEmpty() && this.Server.TrafficLimitStatus != nil && this.Server.TrafficLimitStatus.IsValid() {
|
||||
if this.ReqServer.TrafficLimit != nil && this.ReqServer.TrafficLimit.IsOn && !this.ReqServer.TrafficLimit.IsEmpty() && this.ReqServer.TrafficLimitStatus != nil && this.ReqServer.TrafficLimitStatus.IsValid() {
|
||||
this.doTrafficLimit()
|
||||
this.doEnd()
|
||||
return
|
||||
@@ -300,14 +313,14 @@ func (this *HTTPRequest) doEnd() {
|
||||
// 流量统计
|
||||
// TODO 增加是否开启开关
|
||||
// TODO 增加Header统计,考虑从Conn中读取
|
||||
if this.Server != nil {
|
||||
if this.ReqServer != nil {
|
||||
if this.isCached {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, this.writer.sentBodyBytes, 1, 1, 0, 0, this.Server.ShouldCheckTrafficLimit(), this.Server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.sentBodyBytes, this.writer.sentBodyBytes, 1, 1, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
|
||||
} else {
|
||||
if this.isAttack {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 1, this.writer.sentBodyBytes, this.Server.ShouldCheckTrafficLimit(), this.Server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.sentBodyBytes, 0, 1, 0, 1, this.writer.sentBodyBytes, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
|
||||
} else {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 0, 0, this.Server.ShouldCheckTrafficLimit(), this.Server.PlanId())
|
||||
stats.SharedTrafficStatManager.Add(this.ReqServer.Id, this.ReqHost, this.writer.sentBodyBytes, 0, 1, 0, 0, 0, this.ReqServer.ShouldCheckTrafficLimit(), this.ReqServer.PlanId())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -452,6 +465,26 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
|
||||
this.web.RequestLimit = web.RequestLimit
|
||||
}
|
||||
|
||||
// request scripts
|
||||
if web.RequestScripts != nil {
|
||||
if this.web.RequestScripts == nil {
|
||||
this.web.RequestScripts = web.RequestScripts
|
||||
} else {
|
||||
if web.RequestScripts.InitGroup != nil && (web.RequestScripts.InitGroup.IsPrior || isTop) {
|
||||
if this.web.RequestScripts == nil {
|
||||
this.web.RequestScripts = &serverconfigs.HTTPRequestScriptsConfig{}
|
||||
}
|
||||
this.web.RequestScripts.InitGroup = web.RequestScripts.InitGroup
|
||||
}
|
||||
if web.RequestScripts.RequestGroup != nil && (web.RequestScripts.RequestGroup.IsPrior || isTop) {
|
||||
if this.web.RequestScripts == nil {
|
||||
this.web.RequestScripts = &serverconfigs.HTTPRequestScriptsConfig{}
|
||||
}
|
||||
this.web.RequestScripts.RequestGroup = web.RequestScripts.RequestGroup
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 重写规则
|
||||
if len(web.RewriteRefs) > 0 {
|
||||
for index, ref := range web.RewriteRefs {
|
||||
@@ -521,7 +554,7 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
|
||||
}
|
||||
if varMapping, isMatched := location.Match(rawPath, this.Format); isMatched {
|
||||
// 检查专属域名
|
||||
if len(location.Domains) > 0 && !configutils.MatchDomains(location.Domains, this.Host) {
|
||||
if len(location.Domains) > 0 && !configutils.MatchDomains(location.Domains, this.ReqHost) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -603,11 +636,11 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
if this.IsHTTPS {
|
||||
scheme = "https"
|
||||
}
|
||||
return scheme + "://" + this.Host + this.rawURI
|
||||
return scheme + "://" + this.ReqHost + this.rawURI
|
||||
case "requestPath":
|
||||
return this.requestPath()
|
||||
return this.Path()
|
||||
case "requestPathExtension":
|
||||
return filepath.Ext(this.requestPath())
|
||||
return filepath.Ext(this.Path())
|
||||
case "requestLength":
|
||||
return strconv.FormatInt(this.requestLength(), 10)
|
||||
case "requestTime":
|
||||
@@ -621,7 +654,7 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
}
|
||||
|
||||
if this.web.Root != nil && this.web.Root.IsOn {
|
||||
return filepath.Clean(this.web.Root.Dir + this.requestPath())
|
||||
return filepath.Clean(this.web.Root.Dir + this.Path())
|
||||
}
|
||||
|
||||
return ""
|
||||
@@ -650,7 +683,7 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
case "timestamp":
|
||||
return strconv.FormatInt(this.requestFromTime.Unix(), 10)
|
||||
case "host":
|
||||
return this.Host
|
||||
return this.ReqHost
|
||||
case "referer":
|
||||
return this.RawReq.Referer()
|
||||
case "referer.host":
|
||||
@@ -768,7 +801,7 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
|
||||
// host
|
||||
if prefix == "host" {
|
||||
pieces := strings.Split(this.Host, ".")
|
||||
pieces := strings.Split(this.ReqHost, ".")
|
||||
switch suffix {
|
||||
case "first":
|
||||
if len(pieces) > 0 {
|
||||
@@ -833,6 +866,104 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// geo
|
||||
if prefix == "geo" {
|
||||
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
|
||||
|
||||
switch suffix {
|
||||
case "country.name":
|
||||
if result != nil {
|
||||
return result.Country
|
||||
}
|
||||
return ""
|
||||
case "country.id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
|
||||
}
|
||||
return "0"
|
||||
case "province.name":
|
||||
if result != nil {
|
||||
return result.Province
|
||||
}
|
||||
return ""
|
||||
case "province.id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
|
||||
}
|
||||
return "0"
|
||||
case "city.name":
|
||||
if result != nil {
|
||||
return result.City
|
||||
}
|
||||
return ""
|
||||
case "city.id":
|
||||
if result != nil {
|
||||
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
|
||||
if provinceId > 0 {
|
||||
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
|
||||
} else {
|
||||
return "0"
|
||||
}
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
}
|
||||
|
||||
// ips
|
||||
if prefix == "isp" {
|
||||
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
|
||||
|
||||
switch suffix {
|
||||
case "name":
|
||||
if result != nil {
|
||||
return result.ISP
|
||||
}
|
||||
case "id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// browser
|
||||
if prefix == "browser" {
|
||||
var result = stats.SharedUserAgentParser.Parse(this.RawReq.UserAgent())
|
||||
switch suffix {
|
||||
case "os.name":
|
||||
return result.OS.Name
|
||||
case "os.version":
|
||||
return result.OS.Version
|
||||
case "name":
|
||||
return result.BrowserName
|
||||
case "version":
|
||||
return result.BrowserVersion
|
||||
case "isMobile":
|
||||
if result.IsMobile {
|
||||
return "1"
|
||||
} else {
|
||||
return "0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// product
|
||||
if prefix == "product" {
|
||||
switch suffix {
|
||||
case "name":
|
||||
if sharedNodeConfig.ProductConfig != nil && len(sharedNodeConfig.ProductConfig.Name) > 0 {
|
||||
return sharedNodeConfig.ProductConfig.Name
|
||||
}
|
||||
return teaconst.GlobalProductName
|
||||
case "version":
|
||||
if sharedNodeConfig.ProductConfig != nil && len(sharedNodeConfig.ProductConfig.Version) > 0 {
|
||||
return sharedNodeConfig.ProductConfig.Version
|
||||
}
|
||||
return teaconst.Version
|
||||
}
|
||||
}
|
||||
|
||||
return "${" + varName + "}"
|
||||
})
|
||||
}
|
||||
@@ -846,12 +977,17 @@ func (this *HTTPRequest) addVarMapping(varMapping map[string]string) {
|
||||
|
||||
// 获取请求的客户端地址
|
||||
func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
if supportVar && len(this.remoteAddr) > 0 {
|
||||
return this.remoteAddr
|
||||
}
|
||||
|
||||
if supportVar &&
|
||||
this.web.RemoteAddr != nil &&
|
||||
this.web.RemoteAddr.IsOn &&
|
||||
!this.web.RemoteAddr.IsEmpty() {
|
||||
var remoteAddr = this.Format(this.web.RemoteAddr.Value)
|
||||
if net.ParseIP(remoteAddr) != nil {
|
||||
this.remoteAddr = remoteAddr
|
||||
return remoteAddr
|
||||
}
|
||||
}
|
||||
@@ -864,6 +1000,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
forwardedFor = forwardedFor[:commaIndex]
|
||||
}
|
||||
if net.ParseIP(forwardedFor) != nil {
|
||||
if supportVar {
|
||||
this.remoteAddr = forwardedFor
|
||||
}
|
||||
return forwardedFor
|
||||
}
|
||||
}
|
||||
@@ -873,6 +1012,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
realIP, ok := this.RawReq.Header["X-Real-IP"]
|
||||
if ok && len(realIP) > 0 {
|
||||
if net.ParseIP(realIP[0]) != nil {
|
||||
if supportVar {
|
||||
this.remoteAddr = realIP[0]
|
||||
}
|
||||
return realIP[0]
|
||||
}
|
||||
}
|
||||
@@ -883,6 +1025,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
realIP, ok := this.RawReq.Header["X-Real-Ip"]
|
||||
if ok && len(realIP) > 0 {
|
||||
if net.ParseIP(realIP[0]) != nil {
|
||||
if supportVar {
|
||||
this.remoteAddr = realIP[0]
|
||||
}
|
||||
return realIP[0]
|
||||
}
|
||||
}
|
||||
@@ -892,6 +1037,9 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
remoteAddr := this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err == nil {
|
||||
if supportVar {
|
||||
this.remoteAddr = host
|
||||
}
|
||||
return host
|
||||
} else {
|
||||
return remoteAddr
|
||||
@@ -950,8 +1098,8 @@ func (this *HTTPRequest) requestRemoteUser() string {
|
||||
return username
|
||||
}
|
||||
|
||||
// 请求的URL中路径部分
|
||||
func (this *HTTPRequest) requestPath() string {
|
||||
// Path 请求的URL中路径部分
|
||||
func (this *HTTPRequest) Path() string {
|
||||
uri, err := url.ParseRequestURI(this.rawURI)
|
||||
if err != nil {
|
||||
return ""
|
||||
@@ -1061,9 +1209,148 @@ func (this *HTTPRequest) requestServerPort() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// 获取完整的URL
|
||||
func (this *HTTPRequest) requestFullURL() string {
|
||||
return this.requestScheme() + "://" + this.Host + this.uri
|
||||
func (this *HTTPRequest) Id() string {
|
||||
return this.requestId
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) Server() maps.Map {
|
||||
return maps.Map{"id": this.ReqServer.Id}
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) Node() maps.Map {
|
||||
return maps.Map{"id": teaconst.NodeId}
|
||||
}
|
||||
|
||||
// URL 获取完整的URL
|
||||
func (this *HTTPRequest) URL() string {
|
||||
return this.requestScheme() + "://" + this.ReqHost + this.uri
|
||||
}
|
||||
|
||||
// Host 获取Host
|
||||
func (this *HTTPRequest) Host() string {
|
||||
return this.ReqHost
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) Proto() string {
|
||||
return this.RawReq.Proto
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) ProtoMajor() int {
|
||||
return this.RawReq.ProtoMajor
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) ProtoMinor() int {
|
||||
return this.RawReq.ProtoMinor
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) RemoteAddr() string {
|
||||
return this.requestRemoteAddr(true)
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) RawRemoteAddr() string {
|
||||
addr := this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err == nil {
|
||||
addr = host
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) RemotePort() int {
|
||||
addr := this.RawReq.RemoteAddr
|
||||
_, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return types.Int(port)
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) SetAttr(name string, value string) {
|
||||
this.logAttrs[name] = value
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) SetVar(name string, value string) {
|
||||
this.varMapping[name] = value
|
||||
}
|
||||
|
||||
// ContentLength 请求内容长度
|
||||
func (this *HTTPRequest) ContentLength() int64 {
|
||||
return this.RawReq.ContentLength
|
||||
}
|
||||
|
||||
// Method 请求方法
|
||||
func (this *HTTPRequest) Method() string {
|
||||
return this.RawReq.Method
|
||||
}
|
||||
|
||||
// TransferEncoding 获取传输编码
|
||||
func (this *HTTPRequest) TransferEncoding() string {
|
||||
if len(this.RawReq.TransferEncoding) > 0 {
|
||||
return this.RawReq.TransferEncoding[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Cookie 获取Cookie
|
||||
func (this *HTTPRequest) Cookie(name string) string {
|
||||
c, err := this.RawReq.Cookie(name)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return c.Value
|
||||
}
|
||||
|
||||
// DeleteHeader 删除Header
|
||||
func (this *HTTPRequest) DeleteHeader(name string) {
|
||||
this.RawReq.Header.Del(name)
|
||||
}
|
||||
|
||||
// SetHeader 设置Header
|
||||
func (this *HTTPRequest) SetHeader(name string, values []string) {
|
||||
this.RawReq.Header[name] = values
|
||||
}
|
||||
|
||||
// Header 读取Header
|
||||
func (this *HTTPRequest) Header() http.Header {
|
||||
return this.RawReq.Header
|
||||
}
|
||||
|
||||
// URI 获取当前请求的URI
|
||||
func (this *HTTPRequest) URI() string {
|
||||
return this.uri
|
||||
}
|
||||
|
||||
// SetURI 设置当前请求的URI
|
||||
func (this *HTTPRequest) SetURI(uri string) {
|
||||
this.uri = uri
|
||||
}
|
||||
|
||||
// Done 设置已完成
|
||||
func (this *HTTPRequest) Done() {
|
||||
this.isDone = true
|
||||
}
|
||||
|
||||
// Close 关闭连接
|
||||
func (this *HTTPRequest) Close() {
|
||||
this.Done()
|
||||
|
||||
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if requestConn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
conn, ok := requestConn.(net.Conn)
|
||||
if ok {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Allow 放行
|
||||
func (this *HTTPRequest) Allow() {
|
||||
this.web.FirewallRef = nil
|
||||
}
|
||||
|
||||
// 设置代理相关头部信息
|
||||
@@ -1106,9 +1393,9 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
|
||||
/**{
|
||||
forwarded, ok := header["Forwarded"]
|
||||
if ok {
|
||||
header["Forwarded"] = []string{strings.Join(forwarded, ", ") + ", by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.host + "; proto=" + this.rawScheme}
|
||||
header["Forwarded"] = []string{strings.Join(forwarded, ", ") + ", by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.ReqHost + "; proto=" + this.rawScheme}
|
||||
} else {
|
||||
header["Forwarded"] = []string{"by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.host + "; proto=" + this.rawScheme}
|
||||
header["Forwarded"] = []string{"by=" + this.serverAddr + "; for=" + remoteAddr + "; host=" + this.ReqHost + "; proto=" + this.rawScheme}
|
||||
}
|
||||
}**/
|
||||
|
||||
@@ -1119,7 +1406,7 @@ func (this *HTTPRequest) setForwardHeaders(header http.Header) {
|
||||
|
||||
if this.reverseProxy != nil && this.reverseProxy.ShouldAddXForwardedHostHeader() {
|
||||
if _, ok := header["X-Forwarded-Host"]; !ok {
|
||||
this.RawReq.Header.Set("X-Forwarded-Host", this.Host)
|
||||
this.RawReq.Header.Set("X-Forwarded-Host", this.ReqHost)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1159,7 +1446,7 @@ func (this *HTTPRequest) processRequestHeaders(reqHeader http.Header) {
|
||||
}
|
||||
|
||||
// 域名
|
||||
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.Host) {
|
||||
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.ReqHost) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1243,7 +1530,7 @@ func (this *HTTPRequest) processResponseHeaders(statusCode int) {
|
||||
}
|
||||
|
||||
// 域名
|
||||
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.Host) {
|
||||
if len(header.Domains) > 0 && !configutils.MatchDomains(header.Domains, this.ReqHost) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1277,13 +1564,13 @@ func (this *HTTPRequest) processResponseHeaders(statusCode int) {
|
||||
|
||||
// HSTS
|
||||
if this.IsHTTPS &&
|
||||
this.Server.HTTPS != nil &&
|
||||
this.Server.HTTPS.SSLPolicy != nil &&
|
||||
this.Server.HTTPS.SSLPolicy.IsOn &&
|
||||
this.Server.HTTPS.SSLPolicy.HSTS != nil &&
|
||||
this.Server.HTTPS.SSLPolicy.HSTS.IsOn &&
|
||||
this.Server.HTTPS.SSLPolicy.HSTS.Match(this.Host) {
|
||||
responseHeader.Set(this.Server.HTTPS.SSLPolicy.HSTS.HeaderKey(), this.Server.HTTPS.SSLPolicy.HSTS.HeaderValue())
|
||||
this.ReqServer.HTTPS != nil &&
|
||||
this.ReqServer.HTTPS.SSLPolicy != nil &&
|
||||
this.ReqServer.HTTPS.SSLPolicy.IsOn &&
|
||||
this.ReqServer.HTTPS.SSLPolicy.HSTS != nil &&
|
||||
this.ReqServer.HTTPS.SSLPolicy.HSTS.IsOn &&
|
||||
this.ReqServer.HTTPS.SSLPolicy.HSTS.Match(this.ReqHost) {
|
||||
responseHeader.Set(this.ReqServer.HTTPS.SSLPolicy.HSTS.HeaderKey(), this.ReqServer.HTTPS.SSLPolicy.HSTS.HeaderValue())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1344,22 +1631,6 @@ func (this *HTTPRequest) canIgnore(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// 关闭当前连接
|
||||
func (this *HTTPRequest) closeConn() {
|
||||
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if requestConn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
conn, ok := requestConn.(net.Conn)
|
||||
if ok {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// 检查连接是否已关闭
|
||||
func (this *HTTPRequest) isConnClosed() bool {
|
||||
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
|
||||
@@ -27,7 +27,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
subReq.ProtoMinor = this.RawReq.ProtoMinor
|
||||
subReq.ProtoMajor = this.RawReq.ProtoMajor
|
||||
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
|
||||
subReq.Header.Set("Referer", this.requestFullURL())
|
||||
subReq.Header.Set("Referer", this.URL())
|
||||
var writer = NewEmptyResponseWriter(this.writer)
|
||||
this.doSubRequest(writer, subReq)
|
||||
return writer.StatusCode(), nil
|
||||
@@ -45,7 +45,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.Host
|
||||
headerValue += this.ReqHost
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
this.cacheCanTryStale = false
|
||||
|
||||
cachePolicy := this.Server.HTTPCachePolicy
|
||||
cachePolicy := this.ReqServer.HTTPCachePolicy
|
||||
if cachePolicy == nil || !cachePolicy.IsOn {
|
||||
return
|
||||
}
|
||||
@@ -138,7 +138,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
if err == nil {
|
||||
for _, rpcServerService := range rpcClient.ServerRPCList() {
|
||||
_, err = rpcServerService.PurgeServerCache(rpcClient.Context(), &pb.PurgeServerCacheRequest{
|
||||
Domains: []string{this.Host},
|
||||
Domains: []string{this.ReqHost},
|
||||
Keys: []string{key},
|
||||
Prefixes: nil,
|
||||
})
|
||||
@@ -152,13 +152,19 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
return true
|
||||
}
|
||||
|
||||
// 调用回调
|
||||
this.onRequest()
|
||||
if this.writer.isFinished {
|
||||
return
|
||||
}
|
||||
|
||||
var reader caches.Reader
|
||||
var err error
|
||||
|
||||
// 是否优先检查WebP
|
||||
if this.web.WebP != nil &&
|
||||
this.web.WebP.IsOn &&
|
||||
this.web.WebP.MatchRequest(filepath.Ext(this.requestPath()), this.Format) &&
|
||||
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
|
||||
this.web.WebP.MatchAccept(this.requestHeader("Accept")) {
|
||||
reader, _ = storage.OpenReader(key+webpSuffix, useStale)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ func (this *HTTPRequest) write404() {
|
||||
|
||||
this.processResponseHeaders(http.StatusNotFound)
|
||||
this.writer.WriteHeader(http.StatusNotFound)
|
||||
_, _ = this.writer.Write([]byte("404 page not found: '" + this.requestFullURL() + "'" + " (Request Id: " + this.requestId + ")"))
|
||||
_, _ = this.writer.Write([]byte("404 page not found: '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) writeCode(code int) {
|
||||
@@ -23,7 +23,7 @@ func (this *HTTPRequest) writeCode(code int) {
|
||||
|
||||
this.processResponseHeaders(code)
|
||||
this.writer.WriteHeader(code)
|
||||
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.requestFullURL() + "'" + " (Request Id: " + this.requestId + ")"))
|
||||
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
|
||||
|
||||
11
internal/nodes/http_request_events.go
Normal file
11
internal/nodes/http_request_events.go
Normal file
@@ -0,0 +1,11 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
//go:build !script
|
||||
// +build !script
|
||||
|
||||
package nodes
|
||||
|
||||
func (this *HTTPRequest) onInit() {
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) onRequest() {
|
||||
}
|
||||
@@ -52,13 +52,13 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
}
|
||||
}
|
||||
if !env.Has("SERVER_NAME") {
|
||||
env["SERVER_NAME"] = this.Host
|
||||
env["SERVER_NAME"] = this.ReqHost
|
||||
}
|
||||
if !env.Has("REQUEST_URI") {
|
||||
env["REQUEST_URI"] = this.uri
|
||||
}
|
||||
if !env.Has("HOST") {
|
||||
env["HOST"] = this.Host
|
||||
env["HOST"] = this.ReqHost
|
||||
}
|
||||
|
||||
if len(this.ServerAddr) > 0 {
|
||||
@@ -149,7 +149,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
|
||||
host, found := params["HTTP_HOST"]
|
||||
if !found || len(host) == 0 {
|
||||
params["HTTP_HOST"] = this.Host
|
||||
params["HTTP_HOST"] = this.ReqHost
|
||||
}
|
||||
|
||||
fcgiReq := fcgi.NewRequest()
|
||||
|
||||
@@ -13,7 +13,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
|
||||
if this.web.MergeSlashes {
|
||||
urlPath = utils.CleanPath(urlPath)
|
||||
}
|
||||
fullURL := this.requestScheme() + "://" + this.Host + urlPath
|
||||
fullURL := this.requestScheme() + "://" + this.ReqHost + urlPath
|
||||
for _, u := range this.web.HostRedirects {
|
||||
if !u.IsOn {
|
||||
continue
|
||||
|
||||
@@ -19,9 +19,9 @@ func (this *HTTPRequest) doRequestLimit() (shouldStop bool) {
|
||||
if requestConn != nil {
|
||||
clientConn, ok := requestConn.(ClientConnInterface)
|
||||
if ok && !clientConn.IsBound() {
|
||||
if !clientConn.Bind(this.Server.Id, this.requestRemoteAddr(true), this.web.RequestLimit.MaxConns, this.web.RequestLimit.MaxConnsPerIP) {
|
||||
if !clientConn.Bind(this.ReqServer.Id, this.requestRemoteAddr(true), this.web.RequestLimit.MaxConns, this.web.RequestLimit.MaxConnsPerIP) {
|
||||
this.writeCode(http.StatusTooManyRequests)
|
||||
this.closeConn()
|
||||
this.Close()
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,13 +93,13 @@ func (this *HTTPRequest) log() {
|
||||
accessLog := &pb.HTTPAccessLog{
|
||||
RequestId: this.requestId,
|
||||
NodeId: sharedNodeConfig.Id,
|
||||
ServerId: this.Server.Id,
|
||||
ServerId: this.ReqServer.Id,
|
||||
RemoteAddr: this.requestRemoteAddr(true),
|
||||
RawRemoteAddr: addr,
|
||||
RemotePort: int32(this.requestRemotePort()),
|
||||
RemoteUser: this.requestRemoteUser(),
|
||||
RequestURI: this.rawURI,
|
||||
RequestPath: this.requestPath(),
|
||||
RequestPath: this.Path(),
|
||||
RequestLength: this.requestLength(),
|
||||
RequestTime: this.requestCost,
|
||||
RequestMethod: this.RawReq.Method,
|
||||
@@ -114,7 +114,7 @@ func (this *HTTPRequest) log() {
|
||||
TimeLocal: this.requestFromTime.Format("2/Jan/2006:15:04:05 -0700"),
|
||||
Msec: float64(this.requestFromTime.Unix()) + float64(this.requestFromTime.Nanosecond())/1000000000,
|
||||
Timestamp: this.requestFromTime.Unix(),
|
||||
Host: this.Host,
|
||||
Host: this.ReqHost,
|
||||
Referer: referer,
|
||||
UserAgent: userAgent,
|
||||
Request: this.requestString(),
|
||||
|
||||
@@ -50,7 +50,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) MetricServerId() int64 {
|
||||
return this.Server.Id
|
||||
return this.ReqServer.Id
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) MetricCategory() string {
|
||||
|
||||
@@ -36,12 +36,12 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
requestCall := shared.NewRequestCall()
|
||||
requestCall.Request = this.RawReq
|
||||
requestCall.Formatter = this.Format
|
||||
requestCall.Domain = this.Host
|
||||
requestCall.Domain = this.ReqHost
|
||||
origin := this.reverseProxy.NextOrigin(requestCall)
|
||||
requestCall.CallResponseCallbacks(this.writer)
|
||||
if origin == nil {
|
||||
err := errors.New(this.requestFullURL() + ": no available origin sites for reverse proxy")
|
||||
remotelogs.ServerError(this.Server.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
|
||||
err := errors.New(this.URL() + ": no available origin sites for reverse proxy")
|
||||
remotelogs.ServerError(this.ReqServer.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
return
|
||||
}
|
||||
@@ -60,7 +60,7 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
|
||||
// 处理Scheme
|
||||
if origin.Addr == nil {
|
||||
err := errors.New(this.requestFullURL() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
|
||||
err := errors.New(this.URL() + ": origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
|
||||
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
return
|
||||
@@ -129,7 +129,7 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
this.RawReq.Host = hostname
|
||||
this.RawReq.URL.Host = this.RawReq.Host
|
||||
} else {
|
||||
this.RawReq.URL.Host = this.Host
|
||||
this.RawReq.URL.Host = this.ReqHost
|
||||
}
|
||||
|
||||
// 重组请求URL
|
||||
@@ -147,6 +147,12 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
this.setForwardHeaders(this.RawReq.Header)
|
||||
this.processRequestHeaders(this.RawReq.Header)
|
||||
|
||||
// 调用回调
|
||||
this.onRequest()
|
||||
if this.writer.isFinished {
|
||||
return
|
||||
}
|
||||
|
||||
// 判断是否为Websocket请求
|
||||
if this.RawReq.Header.Get("Upgrade") == "websocket" {
|
||||
this.doWebsocket()
|
||||
@@ -189,7 +195,9 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
} else {
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
}
|
||||
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
|
||||
if httpErr.Err != io.EOF {
|
||||
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+"': "+err.Error())
|
||||
}
|
||||
} else {
|
||||
// 是否为客户端方面的错误
|
||||
isClientError := false
|
||||
@@ -287,6 +295,13 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
this.writer.WriteHeader(resp.StatusCode)
|
||||
}
|
||||
|
||||
// 是否有内容
|
||||
if resp.ContentLength == 0 && len(resp.TransferEncoding) == 0 {
|
||||
_ = resp.Body.Close()
|
||||
this.writer.SetOk()
|
||||
return
|
||||
}
|
||||
|
||||
// 输出到客户端
|
||||
pool := this.bytePool(resp.ContentLength)
|
||||
buf := pool.Get()
|
||||
|
||||
@@ -15,7 +15,7 @@ func (this *HTTPRequest) doRewrite() (shouldShop bool) {
|
||||
if this.rewriteRule.Mode == serverconfigs.HTTPRewriteModeProxy {
|
||||
// 外部URL
|
||||
if this.rewriteIsExternalURL {
|
||||
host := this.Host
|
||||
host := this.ReqHost
|
||||
if len(this.rewriteRule.ProxyHost) > 0 {
|
||||
host = this.rewriteRule.ProxyHost
|
||||
}
|
||||
|
||||
@@ -200,6 +200,12 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
respHeader.Set("ETag", eTag)
|
||||
}
|
||||
|
||||
// 调用回调
|
||||
this.onRequest()
|
||||
if this.writer.isFinished {
|
||||
return
|
||||
}
|
||||
|
||||
// 支持 If-None-Match
|
||||
if this.requestHeader("If-None-Match") == eTag {
|
||||
// 自定义Header
|
||||
|
||||
@@ -6,11 +6,11 @@ import (
|
||||
|
||||
// 统计
|
||||
func (this *HTTPRequest) doStat() {
|
||||
if this.Server == nil {
|
||||
if this.ReqServer == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 内置的统计
|
||||
stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.Server.Id, this.requestRemoteAddr(true), this.writer.SentBodyBytes(), this.isAttack)
|
||||
stats.SharedHTTPRequestStatManager.AddUserAgent(this.Server.Id, this.requestHeader("User-Agent"))
|
||||
stats.SharedHTTPRequestStatManager.AddRemoteAddr(this.ReqServer.Id, this.requestRemoteAddr(true), this.writer.SentBodyBytes(), this.isAttack)
|
||||
stats.SharedHTTPRequestStatManager.AddUserAgent(this.ReqServer.Id, this.requestHeader("User-Agent"))
|
||||
}
|
||||
|
||||
@@ -10,8 +10,8 @@ func (this *HTTPRequest) doSubRequest(writer http.ResponseWriter, rawReq *http.R
|
||||
req := &HTTPRequest{
|
||||
RawReq: rawReq,
|
||||
RawWriter: writer,
|
||||
Server: this.Server,
|
||||
Host: this.Host,
|
||||
ReqServer: this.ReqServer,
|
||||
ReqHost: this.ReqHost,
|
||||
ServerName: this.ServerName,
|
||||
ServerAddr: this.ServerAddr,
|
||||
IsHTTP: this.IsHTTP,
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
// 流量限制
|
||||
func (this *HTTPRequest) doTrafficLimit() {
|
||||
var config = this.Server.TrafficLimit
|
||||
var config = this.ReqServer.TrafficLimit
|
||||
|
||||
this.tags = append(this.tags, "bandwidth")
|
||||
|
||||
|
||||
@@ -17,6 +17,10 @@ import (
|
||||
|
||||
// 调用WAF
|
||||
func (this *HTTPRequest) doWAFRequest() (blocked bool) {
|
||||
if this.web.FirewallRef == nil || !this.web.FirewallRef.IsOn {
|
||||
return
|
||||
}
|
||||
|
||||
var remoteAddr = this.requestRemoteAddr(true)
|
||||
|
||||
// 检查是否为白名单直连
|
||||
@@ -31,16 +35,16 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
|
||||
}
|
||||
|
||||
// 是否在全局名单中
|
||||
if !iplibrary.AllowIP(remoteAddr, this.Server.Id) {
|
||||
if !iplibrary.AllowIP(remoteAddr, this.ReqServer.Id) {
|
||||
this.disableLog = true
|
||||
this.closeConn()
|
||||
this.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// 检查是否在临时黑名单中
|
||||
if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeService, this.Server.Id, remoteAddr) || waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteAddr) {
|
||||
if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeService, this.ReqServer.Id, remoteAddr) || waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteAddr) {
|
||||
this.disableLog = true
|
||||
this.closeConn()
|
||||
this.Close()
|
||||
|
||||
return true
|
||||
}
|
||||
@@ -57,8 +61,8 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
|
||||
}
|
||||
|
||||
// 公用的防火墙设置
|
||||
if this.Server.HTTPFirewallPolicy != nil && this.Server.HTTPFirewallPolicy.IsOn {
|
||||
blocked, breakChecking := this.checkWAFRequest(this.Server.HTTPFirewallPolicy)
|
||||
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
|
||||
blocked, breakChecking := this.checkWAFRequest(this.ReqServer.HTTPFirewallPolicy)
|
||||
if blocked {
|
||||
return true
|
||||
}
|
||||
@@ -208,7 +212,7 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
}
|
||||
|
||||
// 添加统计
|
||||
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Actions)
|
||||
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
|
||||
}
|
||||
|
||||
this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
|
||||
@@ -219,6 +223,10 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
|
||||
// call response waf
|
||||
func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
|
||||
if this.web.FirewallRef == nil || !this.web.FirewallRef.IsOn {
|
||||
return
|
||||
}
|
||||
|
||||
// 当前服务的独立设置
|
||||
if this.web.FirewallPolicy != nil && this.web.FirewallPolicy.IsOn {
|
||||
blocked := this.checkWAFResponse(this.web.FirewallPolicy, resp)
|
||||
@@ -228,8 +236,8 @@ func (this *HTTPRequest) doWAFResponse(resp *http.Response) (blocked bool) {
|
||||
}
|
||||
|
||||
// 公用的防火墙设置
|
||||
if this.Server.HTTPFirewallPolicy != nil && this.Server.HTTPFirewallPolicy.IsOn {
|
||||
blocked := this.checkWAFResponse(this.Server.HTTPFirewallPolicy, resp)
|
||||
if this.ReqServer.HTTPFirewallPolicy != nil && this.ReqServer.HTTPFirewallPolicy.IsOn {
|
||||
blocked := this.checkWAFResponse(this.ReqServer.HTTPFirewallPolicy, resp)
|
||||
if blocked {
|
||||
return true
|
||||
}
|
||||
@@ -266,7 +274,7 @@ func (this *HTTPRequest) checkWAFResponse(firewallPolicy *firewallconfigs.HTTPFi
|
||||
}
|
||||
|
||||
// 添加统计
|
||||
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.Server.Id, this.firewallRuleGroupId, ruleSet.Actions)
|
||||
stats.SharedHTTPRequestStatManager.AddFirewallRuleGroupId(this.ReqServer.Id, this.firewallRuleGroupId, ruleSet.Actions)
|
||||
}
|
||||
|
||||
this.firewallActions = append(ruleSet.ActionCodes(), firewallPolicy.Mode)
|
||||
@@ -313,12 +321,12 @@ func (this *HTTPRequest) WAFRestoreBody(data []byte) {
|
||||
|
||||
// WAFServerId 服务ID
|
||||
func (this *HTTPRequest) WAFServerId() int64 {
|
||||
return this.Server.Id
|
||||
return this.ReqServer.Id
|
||||
}
|
||||
|
||||
// WAFClose 关闭连接
|
||||
func (this *HTTPRequest) WAFClose() {
|
||||
this.closeConn()
|
||||
this.Close()
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -67,7 +69,8 @@ type HTTPWriter struct {
|
||||
cacheWriter caches.Writer // 缓存写入
|
||||
cacheStorage caches.StorageInterface
|
||||
|
||||
isOk bool // 是否完全成功
|
||||
isOk bool // 是否完全成功
|
||||
isFinished bool // 是否已完成
|
||||
}
|
||||
|
||||
// NewHTTPWriter 包装对象
|
||||
@@ -128,6 +131,16 @@ func (this *HTTPWriter) Header() http.Header {
|
||||
return this.writer.Header()
|
||||
}
|
||||
|
||||
// DeleteHeader 删除Header
|
||||
func (this *HTTPWriter) DeleteHeader(name string) {
|
||||
this.writer.Header().Del(name)
|
||||
}
|
||||
|
||||
// SetHeader 设置Header
|
||||
func (this *HTTPWriter) SetHeader(name string, values []string) {
|
||||
this.writer.Header()[name] = values
|
||||
}
|
||||
|
||||
// AddHeaders 添加一组Header
|
||||
func (this *HTTPWriter) AddHeaders(header http.Header) {
|
||||
if this.writer == nil {
|
||||
@@ -206,6 +219,46 @@ func (this *HTTPWriter) WriteHeader(statusCode int) {
|
||||
this.statusCode = statusCode
|
||||
}
|
||||
|
||||
// Send 直接发送内容,并终止请求
|
||||
func (this *HTTPWriter) Send(status int, body string) {
|
||||
this.WriteHeader(status)
|
||||
_, _ = this.WriteString(body)
|
||||
this.isFinished = true
|
||||
}
|
||||
|
||||
// SendFile 发送文件内容,并终止请求
|
||||
func (this *HTTPWriter) SendFile(status int, path string) (int64, error) {
|
||||
this.WriteHeader(status)
|
||||
this.isFinished = true
|
||||
|
||||
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
return 0, errors.New("open file '" + path + "' failed: " + err.Error())
|
||||
}
|
||||
defer func() {
|
||||
_ = fp.Close()
|
||||
}()
|
||||
|
||||
stat, err := fp.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if stat.IsDir() {
|
||||
return 0, errors.New("open file '" + path + "' failed: it is a directory")
|
||||
}
|
||||
|
||||
var bufPool = this.req.bytePool(stat.Size())
|
||||
var buf = bufPool.Get()
|
||||
defer bufPool.Put(buf)
|
||||
|
||||
written, err := io.CopyBuffer(this, fp, buf)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// StatusCode 读取状态码
|
||||
func (this *HTTPWriter) StatusCode() int {
|
||||
if this.statusCode == 0 {
|
||||
@@ -423,8 +476,8 @@ func (this *HTTPWriter) Close() {
|
||||
StaleAt: expiredAt + int64(this.calculateStaleLife()),
|
||||
HeaderSize: this.cacheWriter.HeaderSize(),
|
||||
BodySize: this.cacheWriter.BodySize(),
|
||||
Host: this.req.Host,
|
||||
ServerId: this.req.Server.Id,
|
||||
Host: this.req.ReqHost,
|
||||
ServerId: this.req.ReqServer.Id,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -456,7 +509,7 @@ func (this *HTTPWriter) prepareWebP(size int64) {
|
||||
if this.req.web != nil &&
|
||||
this.req.web.WebP != nil &&
|
||||
this.req.web.WebP.IsOn &&
|
||||
this.req.web.WebP.MatchResponse(this.Header().Get("Content-Type"), size, filepath.Ext(this.req.requestPath()), this.req.Format) &&
|
||||
this.req.web.WebP.MatchResponse(this.Header().Get("Content-Type"), size, filepath.Ext(this.req.Path()), this.req.Format) &&
|
||||
this.req.web.WebP.MatchAccept(this.req.requestHeader("Accept")) &&
|
||||
atomic.LoadInt64(&webpTotalBufferSize) < webpMaxBufferSize {
|
||||
|
||||
@@ -493,7 +546,7 @@ func (this *HTTPWriter) PrepareCompression(size int64) {
|
||||
}
|
||||
|
||||
// 尺寸和类型
|
||||
if !this.compressionConfig.MatchResponse(this.Header().Get("Content-Type"), size, filepath.Ext(this.req.requestPath()), this.req.Format) {
|
||||
if !this.compressionConfig.MatchResponse(this.Header().Get("Content-Type"), size, filepath.Ext(this.req.Path()), this.req.Format) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -548,7 +601,7 @@ func (this *HTTPWriter) prepareCache(size int64) {
|
||||
return
|
||||
}
|
||||
|
||||
cachePolicy := this.req.Server.HTTPCachePolicy
|
||||
cachePolicy := this.req.ReqServer.HTTPCachePolicy
|
||||
if cachePolicy == nil || !cachePolicy.IsOn {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func (this *Listener) listenTCP() error {
|
||||
return err
|
||||
}
|
||||
var netListener = NewClientListener(tcpListener, protocol.IsHTTPFamily() || protocol.IsHTTPSFamily())
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("LISTENER", "quit "+this.group.FullAddr())
|
||||
_ = netListener.Close()
|
||||
})
|
||||
@@ -122,7 +122,7 @@ func (this *Listener) listenUDP() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("LISTENER", "quit "+this.group.FullAddr())
|
||||
_ = listener.Close()
|
||||
})
|
||||
@@ -143,6 +143,8 @@ func (this *Listener) listenUDP() error {
|
||||
}
|
||||
|
||||
func (this *Listener) Close() error {
|
||||
events.Remove(this)
|
||||
|
||||
if this.listener == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ func (this *BaseListener) Reset() {
|
||||
|
||||
}
|
||||
|
||||
// CountActiveListeners 获取当前活跃连接数
|
||||
func (this *BaseListener) CountActiveListeners() int {
|
||||
// CountActiveConnections 获取当前活跃连接数
|
||||
func (this *BaseListener) CountActiveConnections() int {
|
||||
return types.Int(this.countActiveConnections)
|
||||
}
|
||||
|
||||
|
||||
@@ -208,8 +208,8 @@ func (this *HTTPListener) ServeHTTP(rawWriter http.ResponseWriter, rawReq *http.
|
||||
req := &HTTPRequest{
|
||||
RawReq: rawReq,
|
||||
RawWriter: rawWriter,
|
||||
Server: server,
|
||||
Host: reqHost,
|
||||
ReqServer: server,
|
||||
ReqHost: reqHost,
|
||||
ServerName: serverName,
|
||||
ServerAddr: this.addr,
|
||||
IsHTTP: this.isHTTP,
|
||||
|
||||
@@ -16,6 +16,6 @@ type ListenerInterface interface {
|
||||
// Reload 重载配置
|
||||
Reload(serverGroup *serverconfigs.ServerAddressGroup)
|
||||
|
||||
// CountActiveListeners 获取当前活跃的连接数
|
||||
CountActiveListeners() int
|
||||
// CountActiveConnections 获取当前活跃的连接数
|
||||
CountActiveConnections() int
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -29,6 +30,8 @@ type ListenerManager struct {
|
||||
|
||||
retryListenerMap map[string]*Listener // 需要重试的监听器 addr => Listener
|
||||
ticker *time.Ticker
|
||||
|
||||
lastPortStrings string
|
||||
}
|
||||
|
||||
// NewListenerManager 获取新对象
|
||||
@@ -143,6 +146,9 @@ func (this *ListenerManager) Start(node *nodeconfigs.NodeConfig) error {
|
||||
}
|
||||
}
|
||||
|
||||
// 加入到firewalld
|
||||
this.addToFirewalld(groupAddrs)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -153,7 +159,7 @@ func (this *ListenerManager) TotalActiveConnections() int {
|
||||
|
||||
total := 0
|
||||
for _, listener := range this.listenersMap {
|
||||
total += listener.listener.CountActiveListeners()
|
||||
total += listener.listener.CountActiveConnections()
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -214,3 +220,63 @@ func (this *ListenerManager) findProcessNameWithPort(isUdp bool, port string) st
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (this *ListenerManager) addToFirewalld(groupAddrs []string) {
|
||||
if !sharedNodeConfig.AutoOpenPorts {
|
||||
return
|
||||
}
|
||||
|
||||
// 组合端口号
|
||||
var ports = []string{}
|
||||
for _, addr := range groupAddrs {
|
||||
var protocol = "tcp"
|
||||
if strings.HasPrefix(addr, "udp") {
|
||||
protocol = "udp"
|
||||
}
|
||||
|
||||
var lastIndex = strings.LastIndex(addr, ":")
|
||||
if lastIndex > 0 {
|
||||
var portString = addr[lastIndex+1:]
|
||||
ports = append(ports, portString+"/"+protocol)
|
||||
}
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// 检查是否有变化
|
||||
sort.Strings(ports)
|
||||
var newPortStrings = strings.Join(ports, ",")
|
||||
if newPortStrings == this.lastPortStrings {
|
||||
return
|
||||
}
|
||||
this.lastPortStrings = newPortStrings
|
||||
|
||||
firewallCmd, err := exec.LookPath("firewall-cmd")
|
||||
if err != nil || len(firewallCmd) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
remotelogs.Println("FIREWALLD", "open ports automatically")
|
||||
for _, port := range ports {
|
||||
{
|
||||
// TODO 需要支持sudo
|
||||
var cmd = exec.Command(firewallCmd, "--add-port="+port, "--permanent")
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
remotelogs.Warn("FIREWALLD", "'"+cmd.String()+"': "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// TODO 需要支持sudo
|
||||
var cmd = exec.Command(firewallCmd, "--add-port="+port)
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
remotelogs.Warn("FIREWALLD", "'"+cmd.String()+"': "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/configs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/metrics"
|
||||
@@ -61,6 +62,7 @@ func NewNode() *Node {
|
||||
return &Node{
|
||||
sock: gosock.NewTmpSock(teaconst.ProcessName),
|
||||
maxThreads: -1,
|
||||
maxCPU: -1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -449,7 +451,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
|
||||
func (this *Node) startSyncTimer() {
|
||||
// TODO 这个时间间隔可以自行设置
|
||||
ticker := time.NewTicker(60 * time.Second)
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("NODE", "quit sync timer")
|
||||
ticker.Stop()
|
||||
})
|
||||
@@ -635,6 +637,51 @@ func (this *Node) listenSock() error {
|
||||
"limiter": sharedConnectionsLimiter.Len(),
|
||||
},
|
||||
})
|
||||
case "dropIP":
|
||||
var m = maps.NewMap(cmd.Params)
|
||||
var ip = m.GetString("ip")
|
||||
var timeSeconds = m.GetInt("timeoutSeconds")
|
||||
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds)
|
||||
if err != nil {
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
},
|
||||
})
|
||||
} else {
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
case "rejectIP":
|
||||
var m = maps.NewMap(cmd.Params)
|
||||
var ip = m.GetString("ip")
|
||||
var timeSeconds = m.GetInt("timeoutSeconds")
|
||||
err := firewalls.Firewall().RejectSourceIP(ip, timeSeconds)
|
||||
if err != nil {
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
},
|
||||
})
|
||||
} else {
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
case "removeIP":
|
||||
var m = maps.NewMap(cmd.Params)
|
||||
var ip = m.GetString("ip")
|
||||
err := firewalls.Firewall().RemoveSourceIP(ip)
|
||||
if err != nil {
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
},
|
||||
})
|
||||
} else {
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
case "gc":
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -644,7 +691,7 @@ func (this *Node) listenSock() error {
|
||||
}
|
||||
})
|
||||
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
logs.Println("NODE", "quit unix sock")
|
||||
_ = this.sock.Close()
|
||||
})
|
||||
@@ -660,8 +707,9 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
|
||||
runtime.GOMAXPROCS(int(config.MaxCPU))
|
||||
remotelogs.Println("NODE", "[CPU]set max cpu to '"+types.String(config.MaxCPU)+"'")
|
||||
} else {
|
||||
runtime.GOMAXPROCS(runtime.NumCPU())
|
||||
remotelogs.Println("NODE", "[CPU]set max cpu to '"+types.String(runtime.NumCPU())+"'")
|
||||
var threads = runtime.NumCPU() * 4
|
||||
runtime.GOMAXPROCS(threads)
|
||||
remotelogs.Println("NODE", "[CPU]set max cpu to '"+types.String(threads)+"'")
|
||||
}
|
||||
|
||||
this.maxCPU = config.MaxCPU
|
||||
@@ -707,4 +755,9 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
|
||||
time.Local = location
|
||||
this.timezone = timeZone
|
||||
}
|
||||
|
||||
// product information
|
||||
if config.ProductConfig != nil {
|
||||
teaconst.GlobalProductName = config.ProductConfig.Name
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,9 +41,9 @@ func (this *NodeStatusExecutor) Listen() {
|
||||
this.update()
|
||||
|
||||
// TODO 这个时间间隔可以配置
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
var ticker = time.NewTicker(30 * time.Second)
|
||||
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("NODE_STATUS", "quit executor")
|
||||
ticker.Stop()
|
||||
})
|
||||
|
||||
@@ -21,6 +21,9 @@ func init() {
|
||||
SharedOriginStateManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedOriginStateManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// OriginStateManager 源站状态管理
|
||||
@@ -41,7 +44,7 @@ func NewOriginStateManager() *OriginStateManager {
|
||||
|
||||
// Start 启动
|
||||
func (this *OriginStateManager) Start() {
|
||||
events.On(events.EventReload, func() {
|
||||
events.OnKey(events.EventReload, this, func() {
|
||||
this.locker.Lock()
|
||||
this.stateMap = map[int64]*OriginState{}
|
||||
this.locker.Unlock()
|
||||
@@ -58,6 +61,12 @@ func (this *OriginStateManager) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *OriginStateManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// Loop 单次循环检查
|
||||
func (this *OriginStateManager) Loop() error {
|
||||
if sharedNodeConfig == nil {
|
||||
|
||||
@@ -44,10 +44,22 @@ func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.C
|
||||
// TODO 支持TCP4/TCP6
|
||||
// TODO 支持指定特定网卡
|
||||
// TODO Addr支持端口范围,如果有多个端口时,随机一个端口使用
|
||||
// TODO 支持使用证书
|
||||
conn, err = tls.DialWithDialer(&dialer, "tcp", origin.Addr.Host+":"+origin.Addr.PortRange, &tls.Config{
|
||||
|
||||
var tlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
})
|
||||
}
|
||||
if origin.Cert != nil {
|
||||
var obj = origin.Cert.CertObject()
|
||||
if obj != nil {
|
||||
tlsConfig.InsecureSkipVerify = false
|
||||
tlsConfig.Certificates = []tls.Certificate{*obj}
|
||||
if len(origin.Cert.ServerName) > 0 {
|
||||
tlsConfig.ServerName = origin.Cert.ServerName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
conn, err = tls.DialWithDialer(&dialer, "tcp", origin.Addr.Host+":"+origin.Addr.PortRange, tlsConfig)
|
||||
}
|
||||
|
||||
// TODO 需要在合适的时机删除TOA记录
|
||||
@@ -69,10 +81,22 @@ func OriginConnect(origin *serverconfigs.OriginConfig, remoteAddr string) (net.C
|
||||
// TODO 支持TCP4/TCP6
|
||||
// TODO 支持指定特定网卡
|
||||
// TODO Addr支持端口范围,如果有多个端口时,随机一个端口使用
|
||||
// TODO 支持使用证书
|
||||
return tls.Dial("tcp", origin.Addr.Host+":"+origin.Addr.PortRange, &tls.Config{
|
||||
|
||||
var tlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
})
|
||||
}
|
||||
if origin.Cert != nil {
|
||||
var obj = origin.Cert.CertObject()
|
||||
if obj != nil {
|
||||
tlsConfig.InsecureSkipVerify = false
|
||||
tlsConfig.Certificates = []tls.Certificate{*obj}
|
||||
if len(origin.Cert.ServerName) > 0 {
|
||||
tlsConfig.ServerName = origin.Cert.ServerName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tls.Dial("tcp", origin.Addr.Host+":"+origin.Addr.PortRange, tlsConfig)
|
||||
case serverconfigs.ProtocolUDP:
|
||||
addr, err := net.ResolveUDPAddr("udp", origin.Addr.Host+":"+origin.Addr.PortRange)
|
||||
if err != nil {
|
||||
|
||||
@@ -28,7 +28,7 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
// 系统服务管理
|
||||
// SystemServiceManager 系统服务管理
|
||||
type SystemServiceManager struct {
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
@@ -21,17 +20,22 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var sharedSyncAPINodesTask = NewSyncAPINodesTask()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
task := NewSyncAPINodesTask()
|
||||
goman.New(func() {
|
||||
task.Start()
|
||||
sharedSyncAPINodesTask.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
sharedSyncAPINodesTask.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// SyncAPINodesTask API节点同步任务
|
||||
type SyncAPINodesTask struct {
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
func NewSyncAPINodesTask() *SyncAPINodesTask {
|
||||
@@ -39,16 +43,12 @@ func NewSyncAPINodesTask() *SyncAPINodesTask {
|
||||
}
|
||||
|
||||
func (this *SyncAPINodesTask) Start() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
this.ticker = time.NewTicker(5 * time.Minute)
|
||||
if Tea.IsTesting() {
|
||||
// 快速测试
|
||||
ticker = time.NewTicker(1 * time.Minute)
|
||||
this.ticker = time.NewTicker(1 * time.Minute)
|
||||
}
|
||||
events.On(events.EventQuit, func() {
|
||||
remotelogs.Println("SYNC_API_NODES_TASK", "quit task")
|
||||
ticker.Stop()
|
||||
})
|
||||
for range ticker.C {
|
||||
for range this.ticker.C {
|
||||
err := this.Loop()
|
||||
if err != nil {
|
||||
logs.Println("[TASK][SYNC_API_NODES_TASK]" + err.Error())
|
||||
@@ -56,6 +56,12 @@ func (this *SyncAPINodesTask) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *SyncAPINodesTask) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *SyncAPINodesTask) Loop() error {
|
||||
var tr = trackers.Begin("SYNC_API_NODES")
|
||||
defer tr.End()
|
||||
|
||||
@@ -32,14 +32,13 @@ func (this *WAFManager) UpdatePolicies(policies []*firewallconfigs.HTTPFirewallP
|
||||
m := map[int64]*waf.WAF{}
|
||||
for _, p := range policies {
|
||||
w, err := this.convertWAF(p)
|
||||
if w != nil {
|
||||
m[p.Id] = w
|
||||
}
|
||||
if err != nil {
|
||||
remotelogs.Error("WAF", "initialize policy '"+strconv.FormatInt(p.Id, 10)+"' failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
if w == nil {
|
||||
continue
|
||||
}
|
||||
m[p.Id] = w
|
||||
}
|
||||
this.mapping = m
|
||||
}
|
||||
@@ -61,10 +60,12 @@ func (this *WAFManager) convertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
|
||||
policy.Mode = firewallconfigs.FirewallModeDefend
|
||||
}
|
||||
w := &waf.WAF{
|
||||
Id: policy.Id,
|
||||
IsOn: policy.IsOn,
|
||||
Name: policy.Name,
|
||||
Mode: policy.Mode,
|
||||
Id: policy.Id,
|
||||
IsOn: policy.IsOn,
|
||||
Name: policy.Name,
|
||||
Mode: policy.Mode,
|
||||
UseLocalFirewall: policy.UseLocalFirewall,
|
||||
SYNFlood: policy.SYNFlood,
|
||||
}
|
||||
|
||||
// inbound
|
||||
@@ -181,9 +182,9 @@ func (this *WAFManager) convertWAF(policy *firewallconfigs.HTTPFirewallPolicy) (
|
||||
}
|
||||
}
|
||||
|
||||
err := w.Init()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
errorList := w.Init()
|
||||
if len(errorList) > 0 {
|
||||
return w, errorList[0]
|
||||
}
|
||||
|
||||
return w, nil
|
||||
|
||||
254
internal/re/regexp.go
Normal file
254
internal/re/regexp.go
Normal file
@@ -0,0 +1,254 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package re
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var prefixReg = regexp.MustCompile(`^\(\?([\w\s]+)\)`) // (?x)
|
||||
var prefixReg2 = regexp.MustCompile(`^\(\?([\w\s]*:)`) // (?x: ...
|
||||
var braceZero = regexp.MustCompile(`^{\s*0*\s*}`) // {0}
|
||||
var braceZero2 = regexp.MustCompile(`^{\s*0*\s*,`) // {0, x}
|
||||
|
||||
type Regexp struct {
|
||||
exp string
|
||||
rawRegexp *regexp.Regexp
|
||||
|
||||
isStrict bool
|
||||
isCaseInsensitive bool
|
||||
keywords []string
|
||||
keywordsMap RuneMap
|
||||
}
|
||||
|
||||
func MustCompile(exp string) *Regexp {
|
||||
var reg = &Regexp{
|
||||
exp: exp,
|
||||
rawRegexp: regexp.MustCompile(exp),
|
||||
}
|
||||
reg.init()
|
||||
return reg
|
||||
}
|
||||
|
||||
func Compile(exp string) (*Regexp, error) {
|
||||
reg, err := regexp.Compile(exp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewRegexp(reg), nil
|
||||
}
|
||||
|
||||
func NewRegexp(rawRegexp *regexp.Regexp) *Regexp {
|
||||
var reg = &Regexp{
|
||||
exp: rawRegexp.String(),
|
||||
rawRegexp: rawRegexp,
|
||||
}
|
||||
reg.init()
|
||||
return reg
|
||||
}
|
||||
|
||||
func (this *Regexp) init() {
|
||||
if len(this.exp) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
//var keywords = []string{}
|
||||
|
||||
var exp = strings.TrimSpace(this.exp)
|
||||
|
||||
// 去掉前面的(?...)
|
||||
if prefixReg.MatchString(exp) {
|
||||
var matches = prefixReg.FindStringSubmatch(exp)
|
||||
var modifiers = matches[1]
|
||||
if strings.Contains(modifiers, "i") {
|
||||
this.isCaseInsensitive = true
|
||||
}
|
||||
exp = exp[len(matches[0]):]
|
||||
}
|
||||
|
||||
var keywords = this.ParseKeywords(exp)
|
||||
this.keywords = keywords
|
||||
if len(keywords) > 0 {
|
||||
this.keywordsMap = NewRuneTree(keywords)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Regexp) Keywords() []string {
|
||||
return this.keywords
|
||||
}
|
||||
|
||||
func (this *Regexp) Raw() *regexp.Regexp {
|
||||
return this.rawRegexp
|
||||
}
|
||||
|
||||
func (this *Regexp) IsCaseInsensitive() bool {
|
||||
return this.isCaseInsensitive
|
||||
}
|
||||
|
||||
func (this *Regexp) MatchString(s string) bool {
|
||||
if this.keywordsMap != nil {
|
||||
var b = this.keywordsMap.Lookup(s, this.isCaseInsensitive)
|
||||
if !b {
|
||||
return false
|
||||
}
|
||||
if this.isStrict {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return this.rawRegexp.MatchString(s)
|
||||
}
|
||||
|
||||
func (this *Regexp) Match(s []byte) bool {
|
||||
if this.keywordsMap != nil {
|
||||
var b = this.keywordsMap.Lookup(string(s), this.isCaseInsensitive)
|
||||
if !b {
|
||||
return false
|
||||
}
|
||||
if this.isStrict {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return this.rawRegexp.Match(s)
|
||||
}
|
||||
|
||||
// ParseKeywords 提取表达式中的关键词
|
||||
// TODO 支持嵌套,类似于 A(abc|bcd)
|
||||
// TODO 支持 (?:xxx)
|
||||
// TODO 支持 (abc)(bcd)(efg)
|
||||
func (this *Regexp) ParseKeywords(exp string) []string {
|
||||
var keywords = []string{}
|
||||
if len(exp) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var runes = []rune(exp)
|
||||
|
||||
// (a|b|c)
|
||||
reg, err := regexp.Compile(exp)
|
||||
if err == nil {
|
||||
var countSub = reg.NumSubexp()
|
||||
if countSub == 1 {
|
||||
beginIndex := this.indexOfSymbol(runes, '(')
|
||||
if beginIndex >= 0 {
|
||||
runes = runes[beginIndex+1:]
|
||||
symbolIndex := this.indexOfSymbol(runes, ')')
|
||||
if symbolIndex > 0 && this.isPlain(runes[symbolIndex+1:]) {
|
||||
runes = runes[:symbolIndex]
|
||||
if len(runes) == 0 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var lastIndex = 0
|
||||
for index, r := range runes {
|
||||
if r == '|' {
|
||||
if index > 0 && runes[index-1] != '\\' {
|
||||
var ks = this.parseKeyword(runes[lastIndex:index])
|
||||
if len(ks) > 0 {
|
||||
keywords = append(keywords, string(ks))
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
lastIndex = index + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
if lastIndex == 0 {
|
||||
var ks = this.parseKeyword(runes)
|
||||
if len(ks) > 0 {
|
||||
keywords = append(keywords, string(ks))
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
} else if lastIndex > 0 {
|
||||
var ks = this.parseKeyword(runes[lastIndex:])
|
||||
if len(ks) > 0 {
|
||||
keywords = append(keywords, string(ks))
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
func (this *Regexp) parseKeyword(keyword []rune) (result []rune) {
|
||||
if len(keyword) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// remove first \b
|
||||
for index, r := range keyword {
|
||||
if r == '\b' {
|
||||
keyword = keyword[index+1:]
|
||||
break
|
||||
} else if r != '\t' && r != '\r' && r != '\n' && r != ' ' {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(keyword) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for index, r := range keyword {
|
||||
if index == 0 && r == '^' {
|
||||
continue
|
||||
}
|
||||
if r == '(' || r == ')' {
|
||||
if index == 0 {
|
||||
return nil
|
||||
}
|
||||
if keyword[index-1] != '\\' {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if r == '[' || r == '{' || r == '.' || r == '+' || r == '$' {
|
||||
if index == 0 {
|
||||
return nil
|
||||
}
|
||||
if keyword[index-1] != '\\' {
|
||||
if r == '{' && (braceZero.MatchString(string(keyword[index:])) || braceZero2.MatchString(string(keyword[index:]))) { // r {0, ...}
|
||||
return result[:len(result)-1]
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
if r == '?' || r == '*' {
|
||||
if index == 0 {
|
||||
return nil
|
||||
}
|
||||
return result[:len(result)-1]
|
||||
}
|
||||
if r == '\\' || r == '\b' {
|
||||
// TODO 将来更精细的处理 \d, \s, \$等
|
||||
break
|
||||
}
|
||||
|
||||
result = append(result, r)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 查找符号位置
|
||||
func (this *Regexp) indexOfSymbol(runes []rune, symbol rune) int {
|
||||
for index, c := range runes {
|
||||
if c == symbol && (index == 0 || runes[index-1] != '\\') {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// 是否可视为为普通字符
|
||||
func (this *Regexp) isPlain(runes []rune) bool {
|
||||
for _, r := range []rune{'|', '(', ')'} {
|
||||
if this.indexOfSymbol(runes, r) >= 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
120
internal/re/regexp_test.go
Normal file
120
internal/re/regexp_test.go
Normal file
@@ -0,0 +1,120 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package re_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/re"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRegexp(t *testing.T) {
|
||||
for _, s := range []string{"(?i)(abc|efg)", "abc|efg", "abc(.+)"} {
|
||||
var reg = regexp.MustCompile(s)
|
||||
t.Log("===" + s + "===")
|
||||
t.Log(reg.LiteralPrefix())
|
||||
t.Log(reg.NumSubexp())
|
||||
t.Log(reg.SubexpNames())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegexp_MatchString(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
{
|
||||
var r = re.MustCompile("abc")
|
||||
a.IsTrue(r.MatchString("abc"))
|
||||
a.IsFalse(r.MatchString("ab"))
|
||||
}
|
||||
|
||||
{
|
||||
var r = re.MustCompile("(?i)abc|def|ghi")
|
||||
a.IsTrue(r.MatchString("DEF"))
|
||||
a.IsFalse(r.MatchString("ab"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegexp_Sub(t *testing.T) {
|
||||
{
|
||||
reg := regexp.MustCompile(`(a|b|c)(e|f|g)`)
|
||||
for _, subName := range reg.SubexpNames() {
|
||||
t.Log(subName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegexp_ParseKeywords(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var r = re.MustCompile("")
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)def"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)|(?:def)"), []string{}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)|def"), []string{}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("(?i:abc)"), []string{}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("\babc"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords(" \babc"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("\babc\b"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("\b(abc)"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc|efg|hij"), []string{"abc", "efg", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg*|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg?|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg+|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg{2,10}|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg{0,10}|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg.+|hij"), []string{"abc", "hij"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("A(abc|bcd)"), []string{"abc", "bcd"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("^abc"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc$"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\$"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\d"), []string{"abc"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("abc{0,4}"), []string{"ab"}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("{0,4}"), []string{}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("{1,4}"), []string{}))
|
||||
a.IsTrue(testCompareStrings(r.ParseKeywords("中文|北京|上海|golang"), []string{"中文", "北京", "上海", "golang"}))
|
||||
}
|
||||
|
||||
func TestRegexp_ParseKeywords2(t *testing.T) {
|
||||
var r = re.MustCompile("")
|
||||
|
||||
var policy = firewallconfigs.HTTPFirewallTemplate()
|
||||
for _, group := range policy.Inbound.Groups {
|
||||
for _, set := range group.Sets {
|
||||
for _, rule := range set.Rules {
|
||||
if rule.Operator == firewallconfigs.HTTPFirewallRuleOperatorMatch || rule.Operator == firewallconfigs.HTTPFirewallRuleOperatorNotMatch {
|
||||
t.Log(set.Name+":", rule.Value, "=>", r.ParseKeywords(rule.Value))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRegexp_MatchString(b *testing.B) {
|
||||
var r = re.MustCompile("(?i)abc|def|ghi")
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRegexp_MatchString2(b *testing.B) {
|
||||
var r = regexp.MustCompile("(?i)abc|def|ghi")
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
}
|
||||
}
|
||||
|
||||
func testCompareStrings(s1 []string, s2 []string) bool {
|
||||
if len(s1) != len(s2) {
|
||||
return false
|
||||
}
|
||||
for index, s := range s1 {
|
||||
if s != s2[index] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
74
internal/re/rune_tree.go
Normal file
74
internal/re/rune_tree.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package re
|
||||
|
||||
type RuneMap map[rune]*RuneTree
|
||||
|
||||
func (this *RuneMap) Lookup(s string, caseInsensitive bool) bool {
|
||||
return this.lookup([]rune(s), caseInsensitive, 0)
|
||||
}
|
||||
|
||||
func (this RuneMap) lookup(runes []rune, caseInsensitive bool, depth int) bool {
|
||||
if len(runes) == 0 {
|
||||
return false
|
||||
}
|
||||
for i, r := range runes {
|
||||
tree, ok := this[r]
|
||||
if !ok {
|
||||
if caseInsensitive {
|
||||
if r >= 'a' && r <= 'z' {
|
||||
r -= 32
|
||||
tree, ok = this[r]
|
||||
} else if r >= 'A' && r <= 'Z' {
|
||||
r += 32
|
||||
tree, ok = this[r]
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
if depth > 0 {
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if tree.IsEnd {
|
||||
return true
|
||||
}
|
||||
b := tree.Children.lookup(runes[i+1:], caseInsensitive, depth+1)
|
||||
if b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type RuneTree struct {
|
||||
Children RuneMap
|
||||
IsEnd bool
|
||||
}
|
||||
|
||||
func NewRuneTree(list []string) RuneMap {
|
||||
var rootMap = RuneMap{}
|
||||
for _, s := range list {
|
||||
if len(s) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var lastMap = rootMap
|
||||
var runes = []rune(s)
|
||||
for index, r := range runes {
|
||||
tree, ok := lastMap[r]
|
||||
if !ok {
|
||||
tree = &RuneTree{
|
||||
Children: RuneMap{},
|
||||
}
|
||||
lastMap[r] = tree
|
||||
}
|
||||
if index == len(runes)-1 {
|
||||
tree.IsEnd = true
|
||||
}
|
||||
lastMap = tree.Children
|
||||
}
|
||||
}
|
||||
return rootMap
|
||||
}
|
||||
47
internal/re/rune_tree_test.go
Normal file
47
internal/re/rune_tree_test.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package re_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/re"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewRuneTree(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var tree = re.NewRuneTree([]string{"abc", "abd", "def", "GHI", "中国", "@"})
|
||||
a.IsTrue(tree.Lookup("ABC", true))
|
||||
a.IsTrue(tree.Lookup("ABC1", true))
|
||||
a.IsTrue(tree.Lookup("1ABC", true))
|
||||
a.IsTrue(tree.Lookup("def", true))
|
||||
a.IsTrue(tree.Lookup("ghI", true))
|
||||
a.IsFalse(tree.Lookup("d ef", true))
|
||||
a.IsFalse(tree.Lookup("de", true))
|
||||
a.IsFalse(tree.Lookup("de f", true))
|
||||
a.IsTrue(tree.Lookup("我是中国人", true))
|
||||
a.IsTrue(tree.Lookup("iwind.liu@gmail.com", true))
|
||||
}
|
||||
|
||||
func BenchmarkRuneMap_Lookup(b *testing.B) {
|
||||
var tree = re.NewRuneTree([]string{"abc", "abd", "def", "ghi", "中国"})
|
||||
for i := 0; i < b.N; i++ {
|
||||
tree.Lookup("我来自中国", true)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRuneMap_Lookup2_NOT_FOUND(b *testing.B) {
|
||||
var tree = re.NewRuneTree([]string{"abc", "abd", "cde", "GHI"})
|
||||
for i := 0; i < b.N; i++ {
|
||||
tree.Lookup("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", true)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRune_Regexp_FOUND(b *testing.B) {
|
||||
var reg = regexp.MustCompile("(?i)abc|abd|cde|GHI")
|
||||
for i := 0; i < b.N; i++ {
|
||||
reg.MatchString("HELLO WORLD ABC 123 456 abc HELLO WORLD HELLO WORLD ABC 123 456 abc HELLO WORLD HELLO WORLD ABC 123 456 abc HELLO WORLD")
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,39 @@ func ServerSuccess(serverId int64, tag string, description string, logType nodec
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ServerLog 打印服务相关日志信息
|
||||
func ServerLog(serverId int64, tag string, description string, logType nodeconfigs.NodeLogType, params maps.Map) {
|
||||
logs.Println("[" + tag + "]" + description)
|
||||
|
||||
// 参数
|
||||
var paramsJSON []byte
|
||||
if len(params) > 0 {
|
||||
p, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
logs.Println("[LOG]" + err.Error())
|
||||
} else {
|
||||
paramsJSON = p
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case logChan <- &pb.NodeLog{
|
||||
Role: teaconst.Role,
|
||||
Tag: tag,
|
||||
Description: description,
|
||||
Level: "info",
|
||||
NodeId: teaconst.NodeId,
|
||||
ServerId: serverId,
|
||||
CreatedAt: time.Now().Unix(),
|
||||
Type: logType,
|
||||
ParamsJSON: paramsJSON,
|
||||
}:
|
||||
default:
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// 上传日志
|
||||
func uploadLogs() error {
|
||||
logList := []*pb.NodeLog{}
|
||||
|
||||
@@ -81,6 +81,14 @@ func (this *RPCClient) RegionProvinceRPC() pb.RegionProvinceServiceClient {
|
||||
return pb.NewRegionProvinceServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionCityRPC() pb.RegionCityServiceClient {
|
||||
return pb.NewRegionCityServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionProviderRPC() pb.RegionProviderServiceClient {
|
||||
return pb.NewRegionProviderServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPListRPC() pb.IPListServiceClient {
|
||||
return pb.NewIPListServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
@@ -10,12 +10,12 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"github.com/mssola/user_agent"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -64,25 +64,26 @@ func NewHTTPRequestStatManager() *HTTPRequestStatManager {
|
||||
// Start 启动
|
||||
func (this *HTTPRequestStatManager) Start() {
|
||||
// 上传请求总数
|
||||
var monitorTicker = time.NewTicker(1 * time.Minute)
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
monitorTicker.Stop()
|
||||
})
|
||||
goman.New(func() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
if this.totalAttackRequests > 0 {
|
||||
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemAttackRequests, maps.Map{"total": this.totalAttackRequests})
|
||||
this.totalAttackRequests = 0
|
||||
}
|
||||
for range monitorTicker.C {
|
||||
if this.totalAttackRequests > 0 {
|
||||
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemAttackRequests, maps.Map{"total": this.totalAttackRequests})
|
||||
this.totalAttackRequests = 0
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
loopTicker := time.NewTicker(1 * time.Second)
|
||||
uploadTicker := time.NewTicker(30 * time.Minute)
|
||||
var loopTicker = time.NewTicker(1 * time.Second)
|
||||
var uploadTicker = time.NewTicker(30 * time.Minute)
|
||||
if Tea.IsTesting() {
|
||||
uploadTicker = time.NewTicker(10 * time.Second) // 在测试环境下缩短Ticker时间,以方便我们调试
|
||||
}
|
||||
remotelogs.Println("HTTP_REQUEST_STAT_MANAGER", "start ...")
|
||||
events.On(events.EventQuit, func() {
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("HTTP_REQUEST_STAT_MANAGER", "quit")
|
||||
loopTicker.Stop()
|
||||
uploadTicker.Stop()
|
||||
@@ -177,7 +178,6 @@ func (this *HTTPRequestStatManager) AddFirewallRuleGroupId(serverId int64, firew
|
||||
// Loop 单个循环
|
||||
func (this *HTTPRequestStatManager) Loop() error {
|
||||
timeout := time.NewTimer(10 * time.Minute) // 执行的最大时间
|
||||
userAgentParser := &user_agent.UserAgent{}
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
@@ -223,8 +223,8 @@ Loop:
|
||||
serverId := userAgentString[:atIndex]
|
||||
userAgent := userAgentString[atIndex+1:]
|
||||
|
||||
userAgentParser.Parse(userAgent)
|
||||
osInfo := userAgentParser.OSInfo()
|
||||
var result = SharedUserAgentParser.Parse(userAgent)
|
||||
var osInfo = result.OS
|
||||
if len(osInfo.Name) > 0 {
|
||||
dotIndex := strings.Index(osInfo.Version, ".")
|
||||
if dotIndex > -1 {
|
||||
@@ -233,7 +233,7 @@ Loop:
|
||||
this.systemMap[serverId+"@"+osInfo.Name+"@"+osInfo.Version]++
|
||||
}
|
||||
|
||||
browser, browserVersion := userAgentParser.Browser()
|
||||
var browser, browserVersion = result.BrowserName, result.BrowserVersion
|
||||
if len(browser) > 0 {
|
||||
dotIndex := strings.Index(browserVersion, ".")
|
||||
if dotIndex > -1 {
|
||||
@@ -320,6 +320,15 @@ func (this *HTTPRequestStatManager) Upload() error {
|
||||
})
|
||||
}
|
||||
|
||||
// 重置数据
|
||||
// 这里需要放到上传数据之前,防止因上传失败而导致统计数据堆积
|
||||
this.cityMap = map[string]*StatItem{}
|
||||
this.providerMap = map[string]int64{}
|
||||
this.systemMap = map[string]int64{}
|
||||
this.browserMap = map[string]int64{}
|
||||
this.dailyFirewallRuleGroupMap = map[string]int64{}
|
||||
|
||||
// 上传数据
|
||||
_, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
|
||||
Month: timeutil.Format("Ym"),
|
||||
Day: timeutil.Format("Ymd"),
|
||||
@@ -330,14 +339,30 @@ func (this *HTTPRequestStatManager) Upload() error {
|
||||
HttpFirewallRuleGroups: pbFirewallRuleGroups,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
// 是否包含了invalid UTF-8
|
||||
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
|
||||
for _, system := range pbSystems {
|
||||
system.Name = utils.ToValidUTF8string(system.Name)
|
||||
}
|
||||
for _, browser := range pbBrowsers {
|
||||
browser.Name = utils.ToValidUTF8string(browser.Name)
|
||||
}
|
||||
|
||||
// 再次尝试
|
||||
_, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
|
||||
Month: timeutil.Format("Ym"),
|
||||
Day: timeutil.Format("Ymd"),
|
||||
RegionCities: pbCities,
|
||||
RegionProviders: pbProviders,
|
||||
Systems: pbSystems,
|
||||
Browsers: pbBrowsers,
|
||||
HttpFirewallRuleGroups: pbFirewallRuleGroups,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 重置数据
|
||||
this.cityMap = map[string]*StatItem{}
|
||||
this.providerMap = map[string]int64{}
|
||||
this.systemMap = map[string]int64{}
|
||||
this.browserMap = map[string]int64{}
|
||||
this.dailyFirewallRuleGroupMap = map[string]int64{}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -56,16 +56,17 @@ func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig)
|
||||
this.configFunc = configFunc
|
||||
|
||||
// 上传请求总数
|
||||
var monitorTicker = time.NewTicker(1 * time.Minute)
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
monitorTicker.Stop()
|
||||
})
|
||||
goman.New(func() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
if this.totalRequests > 0 {
|
||||
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemRequests, maps.Map{"total": this.totalRequests})
|
||||
this.totalRequests = 0
|
||||
}
|
||||
for range monitorTicker.C {
|
||||
if this.totalRequests > 0 {
|
||||
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemRequests, maps.Map{"total": this.totalRequests})
|
||||
this.totalRequests = 0
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// 上传统计数据
|
||||
@@ -74,8 +75,8 @@ func (this *TrafficStatManager) Start(configFunc func() *nodeconfigs.NodeConfig)
|
||||
// 测试环境缩短上传时间,方便我们调试
|
||||
duration = 30 * time.Second
|
||||
}
|
||||
ticker := time.NewTicker(duration)
|
||||
events.On(events.EventQuit, func() {
|
||||
var ticker = time.NewTicker(duration)
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("TRAFFIC_STAT_MANAGER", "quit")
|
||||
ticker.Stop()
|
||||
})
|
||||
@@ -100,8 +101,7 @@ func (this *TrafficStatManager) Add(serverId int64, domain string, bytes int64,
|
||||
|
||||
this.totalRequests++
|
||||
|
||||
timestamp := utils.UnixTime() / 300 * 300
|
||||
|
||||
timestamp := utils.FloorUnixTime(300)
|
||||
key := strconv.FormatInt(timestamp, 10) + strconv.FormatInt(serverId, 10)
|
||||
this.locker.Lock()
|
||||
|
||||
|
||||
92
internal/stats/user_agent_parser.go
Normal file
92
internal/stats/user_agent_parser.go
Normal file
@@ -0,0 +1,92 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/mssola/user_agent"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var SharedUserAgentParser = NewUserAgentParser()
|
||||
|
||||
// UserAgentParser UserAgent解析器
|
||||
type UserAgentParser struct {
|
||||
parser *user_agent.UserAgent
|
||||
|
||||
cacheMap1 map[string]UserAgentParserResult
|
||||
cacheMap2 map[string]UserAgentParserResult
|
||||
maxCacheItems int
|
||||
|
||||
cacheCursor int
|
||||
locker sync.RWMutex
|
||||
}
|
||||
|
||||
func NewUserAgentParser() *UserAgentParser {
|
||||
var parser = &UserAgentParser{
|
||||
parser: &user_agent.UserAgent{},
|
||||
cacheMap1: map[string]UserAgentParserResult{},
|
||||
cacheMap2: map[string]UserAgentParserResult{},
|
||||
cacheCursor: 0,
|
||||
}
|
||||
|
||||
parser.init()
|
||||
return parser
|
||||
}
|
||||
|
||||
func (this *UserAgentParser) init() {
|
||||
var maxCacheItems = 10_000
|
||||
var systemMemory = utils.SystemMemoryGB()
|
||||
if systemMemory >= 16 {
|
||||
maxCacheItems = 40_000
|
||||
} else if systemMemory >= 8 {
|
||||
maxCacheItems = 30_000
|
||||
} else if systemMemory >= 4 {
|
||||
maxCacheItems = 20_000
|
||||
}
|
||||
this.maxCacheItems = maxCacheItems
|
||||
}
|
||||
|
||||
func (this *UserAgentParser) Parse(userAgent string) (result UserAgentParserResult) {
|
||||
// 限制长度
|
||||
if len(userAgent) == 0 || len(userAgent) > 256 {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.RLock()
|
||||
cacheResult, ok := this.cacheMap1[userAgent]
|
||||
if ok {
|
||||
this.locker.RUnlock()
|
||||
return cacheResult
|
||||
}
|
||||
|
||||
cacheResult, ok = this.cacheMap2[userAgent]
|
||||
if ok {
|
||||
this.locker.RUnlock()
|
||||
return cacheResult
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
|
||||
this.locker.Lock()
|
||||
this.parser.Parse(userAgent)
|
||||
result.OS = this.parser.OSInfo()
|
||||
result.BrowserName, result.BrowserVersion = this.parser.Browser()
|
||||
result.IsMobile = this.parser.Mobile()
|
||||
|
||||
if this.cacheCursor == 0 {
|
||||
this.cacheMap1[userAgent] = result
|
||||
if len(this.cacheMap1) >= this.maxCacheItems {
|
||||
this.cacheCursor = 1
|
||||
this.cacheMap2 = map[string]UserAgentParserResult{}
|
||||
}
|
||||
} else {
|
||||
this.cacheMap2[userAgent] = result
|
||||
if len(this.cacheMap2) >= this.maxCacheItems {
|
||||
this.cacheCursor = 0
|
||||
this.cacheMap1 = map[string]UserAgentParserResult{}
|
||||
}
|
||||
}
|
||||
this.locker.Unlock()
|
||||
|
||||
return
|
||||
}
|
||||
12
internal/stats/user_agent_parser_result.go
Normal file
12
internal/stats/user_agent_parser_result.go
Normal file
@@ -0,0 +1,12 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package stats
|
||||
|
||||
import "github.com/mssola/user_agent"
|
||||
|
||||
type UserAgentParserResult struct {
|
||||
OS user_agent.OSInfo
|
||||
BrowserName string
|
||||
BrowserVersion string
|
||||
IsMobile bool
|
||||
}
|
||||
53
internal/stats/user_agent_parser_test.go
Normal file
53
internal/stats/user_agent_parser_test.go
Normal file
@@ -0,0 +1,53 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUserAgentParser_Parse(t *testing.T) {
|
||||
var parser = NewUserAgentParser()
|
||||
for i := 0; i < 4; i ++ {
|
||||
t.Log(parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/1"))
|
||||
t.Log(parser.Parse("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserAgentParser_Parse_Unknown(t *testing.T) {
|
||||
var parser = NewUserAgentParser()
|
||||
t.Log(parser.Parse("Mozilla/5.0 (Wind 10.0; WOW64; rv:49.0) Apple/537.36 (KHTML, like Gecko) Chr/88.0.4324.96 Sa/537.36 Test/1"))
|
||||
t.Log(parser.Parse(""))
|
||||
}
|
||||
|
||||
func TestUserAgentParser_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var parser = NewUserAgentParser()
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 100_000)))
|
||||
}
|
||||
|
||||
runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log("max cache items:", parser.maxCacheItems)
|
||||
t.Log("cache1:", len(parser.cacheMap1), "cache2:", len(parser.cacheMap2), "cache3:", (stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
|
||||
}
|
||||
|
||||
func BenchmarkUserAgentParser_Parse(b *testing.B) {
|
||||
var parser = NewUserAgentParser()
|
||||
for i := 0; i < b.N; i++ {
|
||||
parser.Parse("Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Test/" + types.String(rands.Int(0, 40000)))
|
||||
}
|
||||
b.Log(len(parser.cacheMap1), len(parser.cacheMap2))
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedCache = NewCache()
|
||||
|
||||
// Cache TTL缓存
|
||||
// 最大的缓存时间为30 * 86400
|
||||
// Piece数据结构:
|
||||
|
||||
@@ -63,7 +63,11 @@ func (this *List) Remove(itemId int64) {
|
||||
|
||||
func (this *List) GC(timestamp int64, callback func(itemId int64)) {
|
||||
this.locker.Lock()
|
||||
itemMap := this.gcItems(timestamp)
|
||||
var itemMap = this.gcItems(timestamp)
|
||||
if len(itemMap) == 0 {
|
||||
this.locker.Unlock()
|
||||
return
|
||||
}
|
||||
this.locker.Unlock()
|
||||
|
||||
if callback != nil {
|
||||
|
||||
@@ -165,3 +165,21 @@ func Benchmark_Map_Uint64(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkList_GC(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var lists = []*List{}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
lists = append(lists, NewList())
|
||||
}
|
||||
|
||||
var timestamp = time.Now().Unix()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
for _, list := range lists {
|
||||
list.GC(timestamp, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,13 @@ func init() {
|
||||
SharedFreeHoursManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedFreeHoursManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// FreeHoursManager 计算节点空闲时间
|
||||
// 以便于我们在空闲时间执行高强度的任务,如果清理缓存等
|
||||
// 以便于我们在空闲时间执行高强度的任务,如清理缓存等
|
||||
type FreeHoursManager struct {
|
||||
dayTrafficMap map[int][24]uint64 // day => [ traffic bytes ]
|
||||
lastBytes uint64
|
||||
@@ -32,6 +35,7 @@ type FreeHoursManager struct {
|
||||
count int
|
||||
|
||||
locker sync.Mutex
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
func NewFreeHoursManager() *FreeHoursManager {
|
||||
@@ -39,8 +43,8 @@ func NewFreeHoursManager() *FreeHoursManager {
|
||||
}
|
||||
|
||||
func (this *FreeHoursManager) Start() {
|
||||
var ticker = time.NewTicker(30 * time.Minute)
|
||||
for range ticker.C {
|
||||
this.ticker = time.NewTicker(30 * time.Minute)
|
||||
for range this.ticker.C {
|
||||
this.Update(atomic.LoadUint64(&teaconst.InTrafficBytes))
|
||||
}
|
||||
}
|
||||
@@ -113,6 +117,12 @@ func (this *FreeHoursManager) IsFreeHour() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *FreeHoursManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// 对数组进行排序,并返回权重
|
||||
func (this *FreeHoursManager) sortUintArrayWeights(arr [24]uint64) [24]uint64 {
|
||||
var l = []map[string]interface{}{}
|
||||
|
||||
14
internal/utils/linkedlist/item.go
Normal file
14
internal/utils/linkedlist/item.go
Normal file
@@ -0,0 +1,14 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package linkedlist
|
||||
|
||||
type Item struct {
|
||||
prev *Item
|
||||
next *Item
|
||||
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
func NewItem(value interface{}) *Item {
|
||||
return &Item{Value: value}
|
||||
}
|
||||
93
internal/utils/linkedlist/list.go
Normal file
93
internal/utils/linkedlist/list.go
Normal file
@@ -0,0 +1,93 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package linkedlist
|
||||
|
||||
type List struct {
|
||||
head *Item
|
||||
end *Item
|
||||
count int
|
||||
}
|
||||
|
||||
func NewList() *List {
|
||||
return &List{}
|
||||
}
|
||||
|
||||
func (this *List) Head() *Item {
|
||||
return this.head
|
||||
}
|
||||
|
||||
func (this *List) End() *Item {
|
||||
return this.end
|
||||
}
|
||||
|
||||
func (this *List) Push(item *Item) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 如果已经在末尾了,则do nothing
|
||||
if this.end == item {
|
||||
return
|
||||
}
|
||||
|
||||
if item.prev != nil || item.next != nil || this.head == item {
|
||||
this.Remove(item)
|
||||
}
|
||||
this.add(item)
|
||||
}
|
||||
|
||||
func (this *List) Remove(item *Item) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
if item.prev != nil {
|
||||
item.prev.next = item.next
|
||||
}
|
||||
if item.next != nil {
|
||||
item.next.prev = item.prev
|
||||
}
|
||||
if item == this.head {
|
||||
this.head = item.next
|
||||
}
|
||||
if item == this.end {
|
||||
this.end = item.prev
|
||||
}
|
||||
|
||||
item.prev = nil
|
||||
item.next = nil
|
||||
this.count--
|
||||
}
|
||||
|
||||
func (this *List) Len() int {
|
||||
return this.count
|
||||
}
|
||||
|
||||
func (this *List) Range(f func(item *Item) (goNext bool)) {
|
||||
for e := this.head; e != nil; e = e.next {
|
||||
goNext := f(e)
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *List) Reset() {
|
||||
this.head = nil
|
||||
this.end = nil
|
||||
}
|
||||
|
||||
func (this *List) add(item *Item) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
if this.end != nil {
|
||||
this.end.next = item
|
||||
item.prev = this.end
|
||||
item.next = nil
|
||||
}
|
||||
this.end = item
|
||||
if this.head == nil {
|
||||
this.head = item
|
||||
}
|
||||
this.count++
|
||||
}
|
||||
82
internal/utils/linkedlist/list_test.go
Normal file
82
internal/utils/linkedlist/list_test.go
Normal file
@@ -0,0 +1,82 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package linkedlist_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewList_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var list = linkedlist.NewList()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
var item = &linkedlist.Item{}
|
||||
list.Push(item)
|
||||
}
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
|
||||
t.Log(list.Len())
|
||||
|
||||
var count = 0
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
count++
|
||||
return true
|
||||
})
|
||||
t.Log(count)
|
||||
}
|
||||
|
||||
func TestList_Push(t *testing.T) {
|
||||
var list = linkedlist.NewList()
|
||||
list.Push(linkedlist.NewItem(1))
|
||||
list.Push(linkedlist.NewItem(2))
|
||||
|
||||
var item3 = linkedlist.NewItem(3)
|
||||
list.Push(item3)
|
||||
|
||||
var item4 = linkedlist.NewItem(4)
|
||||
list.Push(item4)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Log("=== after push3 ===")
|
||||
list.Push(item3)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Log("=== after push4 ===")
|
||||
list.Push(item4)
|
||||
list.Push(item3)
|
||||
list.Push(item3)
|
||||
list.Push(item3)
|
||||
list.Push(item4)
|
||||
list.Push(item4)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
|
||||
t.Log("=== after remove ===")
|
||||
list.Remove(item3)
|
||||
list.Range(func(item *linkedlist.Item) (goNext bool) {
|
||||
t.Log(item.Value)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkList_Add(b *testing.B) {
|
||||
var list = linkedlist.NewList()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var item = &linkedlist.Item{}
|
||||
list.Push(item)
|
||||
}
|
||||
}
|
||||
@@ -5,17 +5,17 @@ import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// convert bytes to string
|
||||
// UnsafeBytesToString convert bytes to string
|
||||
func UnsafeBytesToString(bs []byte) string {
|
||||
return *(*string)(unsafe.Pointer(&bs))
|
||||
}
|
||||
|
||||
// convert string to bytes
|
||||
// UnsafeStringToBytes convert string to bytes
|
||||
func UnsafeStringToBytes(s string) []byte {
|
||||
return *(*[]byte)(unsafe.Pointer(&s))
|
||||
}
|
||||
|
||||
// format address
|
||||
// FormatAddress format address
|
||||
func FormatAddress(addr string) string {
|
||||
if strings.HasSuffix(addr, "unix:") {
|
||||
return addr
|
||||
@@ -27,7 +27,7 @@ func FormatAddress(addr string) string {
|
||||
return addr
|
||||
}
|
||||
|
||||
// format address list
|
||||
// FormatAddressList format address list
|
||||
func FormatAddressList(addrList []string) []string {
|
||||
result := []string{}
|
||||
for _, addr := range addrList {
|
||||
@@ -35,3 +35,7 @@ func FormatAddressList(addrList []string) []string {
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func ToValidUTF8string(v string) string {
|
||||
return strings.ToValidUTF8(v, "")
|
||||
}
|
||||
|
||||
@@ -23,6 +23,21 @@ func UnixTime() int64 {
|
||||
return unixTime
|
||||
}
|
||||
|
||||
// FloorUnixTime 取整
|
||||
func FloorUnixTime(seconds int) int64 {
|
||||
return UnixTime() / int64(seconds) * int64(seconds)
|
||||
}
|
||||
|
||||
// CeilUnixTime 取整并加1
|
||||
func CeilUnixTime(seconds int) int64 {
|
||||
return UnixTime()/int64(seconds)*int64(seconds) + int64(seconds)
|
||||
}
|
||||
|
||||
// NextMinuteUnixTime 获取下一分钟开始的时间戳
|
||||
func NextMinuteUnixTime() int64 {
|
||||
return CeilUnixTime(60)
|
||||
}
|
||||
|
||||
// UnixTimeMilli 获取时间戳,精确到毫秒
|
||||
func UnixTimeMilli() int64 {
|
||||
return unixTimeMilli
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -19,3 +20,11 @@ func TestGMTUnixTime(t *testing.T) {
|
||||
func TestGMTTime(t *testing.T) {
|
||||
t.Log(GMTTime(time.Now()))
|
||||
}
|
||||
|
||||
func TestFloorUnixTime(t *testing.T) {
|
||||
var timestamp = time.Now().Unix()
|
||||
t.Log("floor 60:", timestamp, FloorUnixTime(60), timeutil.FormatTime("Y-m-d H:i:s", FloorUnixTime(60)))
|
||||
t.Log("ceil 60:", timestamp, CeilUnixTime(60), timeutil.FormatTime("Y-m-d H:i:s", CeilUnixTime(60)))
|
||||
t.Log("floor 300:", timestamp, FloorUnixTime(300), timeutil.FormatTime("Y-m-d H:i:s", FloorUnixTime(300)))
|
||||
t.Log("next minute:", NextMinuteUnixTime())
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
|
||||
timeout = 60 // 默认封锁60秒
|
||||
}
|
||||
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(timeout), waf.Id, group.Id, set.Id)
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(timeout), waf.Id, waf.UseLocalFirewall, group.Id, set.Id, "")
|
||||
|
||||
if writer != nil {
|
||||
// close the connection
|
||||
@@ -83,6 +83,8 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
|
||||
logs.Error(err)
|
||||
return false
|
||||
}
|
||||
req.Header.Set("User-Agent", teaconst.GlobalProductName+"/"+teaconst.Version)
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
|
||||
@@ -18,7 +18,10 @@ const (
|
||||
)
|
||||
|
||||
type CaptchaAction struct {
|
||||
Life int32 `yaml:"life" json:"life"`
|
||||
Life int32 `yaml:"life" json:"life"`
|
||||
MaxFails int `yaml:"maxFails" json:"maxFails"` // 最大失败次数
|
||||
FailBlockTimeout int `yaml:"failBlockTimeout" json:"failBlockTimeout"` // 失败拦截时间
|
||||
|
||||
Language string `yaml:"language" json:"language"` // 语言,zh-CN, en-US ...
|
||||
AddToWhiteList bool `yaml:"addToWhiteList" json:"addToWhiteList"` // 是否加入到白名单
|
||||
Scope string `yaml:"scope" json:"scope"`
|
||||
@@ -60,12 +63,14 @@ func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
}
|
||||
|
||||
var captchaConfig = maps.Map{
|
||||
"action": this,
|
||||
"timestamp": time.Now().Unix(),
|
||||
"url": refURL,
|
||||
"policyId": waf.Id,
|
||||
"groupId": group.Id,
|
||||
"setId": set.Id,
|
||||
"action": this,
|
||||
"timestamp": time.Now().Unix(),
|
||||
"maxFails": this.MaxFails,
|
||||
"failBlockTimeout": this.FailBlockTimeout,
|
||||
"url": refURL,
|
||||
"policyId": waf.Id,
|
||||
"groupId": group.Id,
|
||||
"setId": set.Id,
|
||||
}
|
||||
info, err := utils.SimpleEncryptMap(captchaConfig)
|
||||
if err != nil {
|
||||
|
||||
@@ -56,7 +56,7 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
life = 600 // 默认10分钟
|
||||
}
|
||||
var setId = m.GetString("setId")
|
||||
SharedIPWhiteList.RecordIP("set:"+setId, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+life, m.GetInt64("policyId"), m.GetInt64("groupId"), m.GetInt64("setId"))
|
||||
SharedIPWhiteList.RecordIP("set:"+setId, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+life, m.GetInt64("policyId"), false, m.GetInt64("groupId"), m.GetInt64("setId"), "")
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,8 @@ type recordIPTask struct {
|
||||
level string
|
||||
serverId int64
|
||||
|
||||
reason string
|
||||
|
||||
sourceServerId int64
|
||||
sourceHTTPFirewallPolicyId int64
|
||||
sourceHTTPFirewallRuleGroupId int64
|
||||
@@ -44,12 +46,16 @@ func init() {
|
||||
if strings.Contains(task.ip, ":") {
|
||||
ipType = "ipv6"
|
||||
}
|
||||
var reason = task.reason
|
||||
if len(reason) == 0 {
|
||||
reason = "触发WAF规则自动加入"
|
||||
}
|
||||
_, err = rpcClient.IPItemRPC().CreateIPItem(rpcClient.Context(), &pb.CreateIPItemRequest{
|
||||
IpListId: task.listId,
|
||||
IpFrom: task.ip,
|
||||
IpTo: "",
|
||||
ExpiredAt: task.expiredAt,
|
||||
Reason: "触发WAF规则自动加入",
|
||||
Reason: reason,
|
||||
Type: ipType,
|
||||
EventLevel: task.level,
|
||||
ServerId: task.serverId,
|
||||
|
||||
@@ -3,6 +3,8 @@ package waf
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/jsonutils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
@@ -54,7 +56,7 @@ func (this *CaptchaValidator) Run(request requests.Request, writer http.Response
|
||||
var originURL = m.GetString("url")
|
||||
|
||||
if request.WAFRaw().Method == http.MethodPost && len(request.WAFRaw().FormValue("GOEDGE_WAF_CAPTCHA_ID")) > 0 {
|
||||
this.validate(actionConfig, m.GetInt64("policyId"), m.GetInt64("groupId"), setId, originURL, request, writer)
|
||||
this.validate(actionConfig, m.GetInt("maxFails"), m.GetInt("failBlockTimeout"), m.GetInt64("policyId"), m.GetInt64("groupId"), setId, originURL, request, writer)
|
||||
} else {
|
||||
this.show(actionConfig, request, writer)
|
||||
}
|
||||
@@ -112,6 +114,7 @@ func (this *CaptchaValidator) show(actionConfig *CaptchaAction, request requests
|
||||
<html>
|
||||
<head>
|
||||
<title>` + msgTitle + `</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=0">
|
||||
<script type="text/javascript">
|
||||
if (window.addEventListener != null) {
|
||||
window.addEventListener("load", function () {
|
||||
@@ -142,23 +145,35 @@ func (this *CaptchaValidator) show(actionConfig *CaptchaAction, request requests
|
||||
</html>`))
|
||||
}
|
||||
|
||||
func (this *CaptchaValidator) validate(actionConfig *CaptchaAction, policyId int64, groupId int64, setId int64, originURL string, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *CaptchaValidator) validate(actionConfig *CaptchaAction, maxFails int, failBlockTimeout int, policyId int64, groupId int64, setId int64, originURL string, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
captchaId := request.WAFRaw().FormValue("GOEDGE_WAF_CAPTCHA_ID")
|
||||
if len(captchaId) > 0 {
|
||||
captchaCode := request.WAFRaw().FormValue("GOEDGE_WAF_CAPTCHA_CODE")
|
||||
if captcha.VerifyString(captchaId, captchaCode) {
|
||||
// 删除计数
|
||||
ttlcache.SharedCache.Delete("CAPTCHA:FAILS:" + request.WAFRemoteIP())
|
||||
|
||||
var life = CaptchaSeconds
|
||||
if actionConfig.Life > 0 {
|
||||
life = types.Int(actionConfig.Life)
|
||||
}
|
||||
|
||||
// 加入到白名单
|
||||
SharedIPWhiteList.RecordIP("set:"+strconv.FormatInt(setId, 10), actionConfig.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(life), policyId, groupId, setId)
|
||||
SharedIPWhiteList.RecordIP("set:"+strconv.FormatInt(setId, 10), actionConfig.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(life), policyId, false, groupId, setId, "")
|
||||
|
||||
http.Redirect(writer, request.WAFRaw(), originURL, http.StatusSeeOther)
|
||||
|
||||
return false
|
||||
} else {
|
||||
// 增加计数
|
||||
if maxFails > 0 && failBlockTimeout > 0 {
|
||||
var countFails = ttlcache.SharedCache.IncreaseInt64("CAPTCHA:FAILS:"+request.WAFRemoteIP(), 1, time.Now().Unix()+300)
|
||||
if int(countFails) >= maxFails {
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, false, groupId, setId, "CAPTCHA验证连续失败")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
http.Redirect(writer, request.WAFRaw(), request.WAFRaw().URL.String(), http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package checkpoints
|
||||
|
||||
// check point definition
|
||||
// CheckpointDefinition check point definition
|
||||
type CheckpointDefinition struct {
|
||||
Name string
|
||||
Description string
|
||||
|
||||
25
internal/waf/checkpoints/request_geo_city_name.go
Normal file
25
internal/waf/checkpoints/request_geo_city_name.go
Normal file
@@ -0,0 +1,25 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package checkpoints
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
)
|
||||
|
||||
type RequestGeoCityNameCheckpoint struct {
|
||||
Checkpoint
|
||||
}
|
||||
|
||||
func (this *RequestGeoCityNameCheckpoint) IsComposed() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *RequestGeoCityNameCheckpoint) RequestValue(req requests.Request, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
value = req.Format("${geo.city.name}")
|
||||
return
|
||||
}
|
||||
|
||||
func (this *RequestGeoCityNameCheckpoint) ResponseValue(req requests.Request, resp *requests.Response, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
return this.RequestValue(req, param, options)
|
||||
}
|
||||
25
internal/waf/checkpoints/request_geo_country_name.go
Normal file
25
internal/waf/checkpoints/request_geo_country_name.go
Normal file
@@ -0,0 +1,25 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package checkpoints
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
)
|
||||
|
||||
type RequestGeoCountryNameCheckpoint struct {
|
||||
Checkpoint
|
||||
}
|
||||
|
||||
func (this *RequestGeoCountryNameCheckpoint) IsComposed() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *RequestGeoCountryNameCheckpoint) RequestValue(req requests.Request, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
value = req.Format("${geo.country.name}")
|
||||
return
|
||||
}
|
||||
|
||||
func (this *RequestGeoCountryNameCheckpoint) ResponseValue(req requests.Request, resp *requests.Response, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
return this.RequestValue(req, param, options)
|
||||
}
|
||||
25
internal/waf/checkpoints/request_geo_province_name.go
Normal file
25
internal/waf/checkpoints/request_geo_province_name.go
Normal file
@@ -0,0 +1,25 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package checkpoints
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
)
|
||||
|
||||
type RequestGeoProvinceNameCheckpoint struct {
|
||||
Checkpoint
|
||||
}
|
||||
|
||||
func (this *RequestGeoProvinceNameCheckpoint) IsComposed() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *RequestGeoProvinceNameCheckpoint) RequestValue(req requests.Request, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
value = req.Format("${geo.province.name}")
|
||||
return
|
||||
}
|
||||
|
||||
func (this *RequestGeoProvinceNameCheckpoint) ResponseValue(req requests.Request, resp *requests.Response, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
return this.RequestValue(req, param, options)
|
||||
}
|
||||
25
internal/waf/checkpoints/request_isp_name.go
Normal file
25
internal/waf/checkpoints/request_isp_name.go
Normal file
@@ -0,0 +1,25 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package checkpoints
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
)
|
||||
|
||||
type RequestISPNameCheckpoint struct {
|
||||
Checkpoint
|
||||
}
|
||||
|
||||
func (this *RequestISPNameCheckpoint) IsComposed() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *RequestISPNameCheckpoint) RequestValue(req requests.Request, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
value = req.Format("${isp.name}")
|
||||
return
|
||||
}
|
||||
|
||||
func (this *RequestISPNameCheckpoint) ResponseValue(req requests.Request, resp *requests.Response, param string, options maps.Map) (value interface{}, sysErr error, userErr error) {
|
||||
return this.RequestValue(req, param, options)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user