Compare commits
158 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b679fc3b25 | ||
|
|
ced82d8a30 | ||
|
|
73108faa3e | ||
|
|
e89460e36f | ||
|
|
cd19a4a7bc | ||
|
|
4dcd47d8bc | ||
|
|
388e7d2683 | ||
|
|
36f9effbf2 | ||
|
|
8bb47fb915 | ||
|
|
7127d26fff | ||
|
|
8fae094161 | ||
|
|
16349041fb | ||
|
|
2793e0de89 | ||
|
|
82a7971718 | ||
|
|
1ef59ccf65 | ||
|
|
cc5d2f1862 | ||
|
|
5bff24b88d | ||
|
|
bc2f8b1e4c | ||
|
|
8935f35b4e | ||
|
|
874694f662 | ||
|
|
27506ae436 | ||
|
|
3d9f40331d | ||
|
|
538f18afb0 | ||
|
|
b4d9fc02bd | ||
|
|
537138c8a1 | ||
|
|
f0d0a39a03 | ||
|
|
bcce2a2767 | ||
|
|
86db3dfc49 | ||
|
|
ac120a728c | ||
|
|
871de0e655 | ||
|
|
fa6be81abe | ||
|
|
1b2f01f0f4 | ||
|
|
09467a4d08 | ||
|
|
a17bbc3df1 | ||
|
|
dc495c70b3 | ||
|
|
d0cf145f85 | ||
|
|
95f1e61489 | ||
|
|
42a0161312 | ||
|
|
729443f0b4 | ||
|
|
3888565c0f | ||
|
|
1ca967534a | ||
|
|
c01bb57dea | ||
|
|
adadb52d4e | ||
|
|
38a7cc17da | ||
|
|
246bb45614 | ||
|
|
b320d2dc58 | ||
|
|
96c63300f4 | ||
|
|
3eaf090aac | ||
|
|
b157448ad2 | ||
|
|
44998a23fb | ||
|
|
e3e30ffee5 | ||
|
|
12bddc6e82 | ||
|
|
771d2d8013 | ||
|
|
3bf94bc032 | ||
|
|
a1aa2b9224 | ||
|
|
8d28ba3426 | ||
|
|
5a72c10d83 | ||
|
|
ec113c59ab | ||
|
|
7f9d95ba37 | ||
|
|
558314265a | ||
|
|
c793dd9d8c | ||
|
|
33e899a008 | ||
|
|
903f511139 | ||
|
|
49dafafdc5 | ||
|
|
c6b01dc10a | ||
|
|
1119b351aa | ||
|
|
7b8ef8e85b | ||
|
|
dd41d88647 | ||
|
|
699cea4382 | ||
|
|
dfb0e60acc | ||
|
|
417e10970a | ||
|
|
6437875979 | ||
|
|
beb251a3cc | ||
|
|
8069f20c77 | ||
|
|
8161270f47 | ||
|
|
be7de223af | ||
|
|
7469f528a8 | ||
|
|
1f88db8829 | ||
|
|
844a5fc321 | ||
|
|
06db07ac4b | ||
|
|
00df3fca84 | ||
|
|
4d679e31bc | ||
|
|
06e27bb469 | ||
|
|
684ba7082b | ||
|
|
8934962de2 | ||
|
|
0cf37f25dc | ||
|
|
d7a6d71fea | ||
|
|
a26f7941d5 | ||
|
|
f5365e5420 | ||
|
|
56d21f867b | ||
|
|
d18a301c61 | ||
|
|
afb937030c | ||
|
|
8faa82c453 | ||
|
|
b17b63aec5 | ||
|
|
c30dbb811f | ||
|
|
a58816361e | ||
|
|
8d37aefd95 | ||
|
|
df7fee966e | ||
|
|
01cfccebbd | ||
|
|
6bd7da5e6e | ||
|
|
dcba9c2f3e | ||
|
|
f38e80e82d | ||
|
|
9bd38094c3 | ||
|
|
7e37fc3b80 | ||
|
|
d775dfeeaa | ||
|
|
0486f86898 | ||
|
|
102157c893 | ||
|
|
dbd92368ae | ||
|
|
9e418e73bf | ||
|
|
5d40eec163 | ||
|
|
1a7a67238d | ||
|
|
6707437bae | ||
|
|
df7859387d | ||
|
|
889c52330d | ||
|
|
0e912b79cd | ||
|
|
12f3916e45 | ||
|
|
7813e2c3d2 | ||
|
|
54eff9bfae | ||
|
|
635cdd4338 | ||
|
|
4c64d3ab0f | ||
|
|
93a5c90fcb | ||
|
|
eb5e863146 | ||
|
|
78e566174f | ||
|
|
dd93a93ba9 | ||
|
|
aaa6899976 | ||
|
|
e04e3287b4 | ||
|
|
e715693156 | ||
|
|
2798c3c5e5 | ||
|
|
489e081720 | ||
|
|
77a8eb5c1a | ||
|
|
3b7d2b91c7 | ||
|
|
20b299fb3b | ||
|
|
e4b3d2b2aa | ||
|
|
d237ee6b5b | ||
|
|
34aa6125df | ||
|
|
24fc2249bb | ||
|
|
84c931b411 | ||
|
|
7f422a2946 | ||
|
|
13194366a5 | ||
|
|
993cda7766 | ||
|
|
a46e970c74 | ||
|
|
085adcf1c4 | ||
|
|
c1af8b36a4 | ||
|
|
8cba12b4b5 | ||
|
|
3debe1d1df | ||
|
|
549f110e5f | ||
|
|
f3a45e9e64 | ||
|
|
f461760158 | ||
|
|
a49b724745 | ||
|
|
0df5dfad23 | ||
|
|
aeb1bc08a7 | ||
|
|
c51aca621a | ||
|
|
c78d055dae | ||
|
|
4502a3b132 | ||
|
|
fa99d86d6f | ||
|
|
2edd2bb105 | ||
|
|
0f8aee0ccb | ||
|
|
2500929a99 |
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*_plus.go
|
||||
*-plus.sh
|
||||
@@ -7,6 +7,7 @@ function build() {
|
||||
DIST=$ROOT/"../dist/${NAME}"
|
||||
OS=${1}
|
||||
ARCH=${2}
|
||||
TAG=${3}
|
||||
|
||||
if [ -z $OS ]; then
|
||||
echo "usage: build.sh OS ARCH"
|
||||
@@ -16,6 +17,9 @@ function build() {
|
||||
echo "usage: build.sh OS ARCH"
|
||||
exit
|
||||
fi
|
||||
if [ -z $TAG ]; then
|
||||
TAG="community"
|
||||
fi
|
||||
|
||||
echo "checking ..."
|
||||
ZIP_PATH=$(which zip)
|
||||
@@ -24,8 +28,8 @@ function build() {
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "building v${VERSION}/${OS}/${ARCH} ..."
|
||||
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
|
||||
echo "building v${VERSION}/${OS}/${ARCH}/${TAG} ..."
|
||||
ZIP="${NAME}-${OS}-${ARCH}-${TAG}-v${VERSION}.zip"
|
||||
|
||||
echo "copying ..."
|
||||
if [ ! -d $DIST ]; then
|
||||
@@ -48,7 +52,42 @@ function build() {
|
||||
fi
|
||||
|
||||
echo "building ..."
|
||||
env GOOS=${OS} GOARCH=${ARCH} go build -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
|
||||
|
||||
MUSL_DIR="/usr/local/opt/musl-cross/bin"
|
||||
CC_PATH=""
|
||||
CXX_PATH=""
|
||||
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
|
||||
# /usr/local/opt/musl-cross/bin/
|
||||
if [ "${ARCH}" == "amd64" ]; then
|
||||
CC_PATH="x86_64-linux-musl-gcc"
|
||||
CXX_PATH="x86_64-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "386" ]; then
|
||||
CC_PATH="i486-linux-musl-gcc"
|
||||
CXX_PATH="i486-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "arm64" ]; then
|
||||
CC_PATH="aarch64-linux-musl-gcc"
|
||||
CXX_PATH="aarch64-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "arm" ]; then
|
||||
CC_PATH="arm-linux-musleabi-gcc"
|
||||
CXX_PATH="arm-linux-musleabi-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "mips64" ]; then
|
||||
CC_PATH="mips64-linux-musl-gcc"
|
||||
CXX_PATH="mips64-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "mips64le" ]; then
|
||||
CC_PATH="mips64el-linux-musl-gcc"
|
||||
CXX_PATH="mips64el-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
if [ ! -z $CC_PATH ]; then
|
||||
env CC=$MUSL_DIR/$CC_PATH CXX=$MUSL_DIR/$CXX_PATH GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" $ROOT/../cmd/edge-node/main.go
|
||||
else
|
||||
env GOOS=${OS} GOARCH=${ARCH} CGO_ENABLED=1 go build -tags $TAG -o $DIST/bin/${NAME} -ldflags="-s -w" $ROOT/../cmd/edge-node/main.go
|
||||
fi
|
||||
|
||||
# delete hidden files
|
||||
find $DIST -name ".DS_Store" -delete
|
||||
@@ -79,4 +118,4 @@ function lookup-version() {
|
||||
fi
|
||||
}
|
||||
|
||||
build $1 $2
|
||||
build $1 $2 $3
|
||||
|
||||
2
build/data/.gitignore
vendored
2
build/data/.gitignore
vendored
@@ -1 +1 @@
|
||||
index.*
|
||||
*
|
||||
1
build/edge-toa/.gitignore
vendored
1
build/edge-toa/.gitignore
vendored
@@ -1 +0,0 @@
|
||||
edge-toa
|
||||
BIN
build/edge-toa/edge-toa
Executable file
BIN
build/edge-toa/edge-toa
Executable file
Binary file not shown.
@@ -9,7 +9,7 @@
|
||||
<h3>403 Forbidden</h3>
|
||||
<p>Sorry, your access to the page has been denied. Please try again later.</p>
|
||||
|
||||
<footer>Powered by TeaEdge.</footer>
|
||||
<footer>Powered by GoEdge.</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -9,7 +9,7 @@
|
||||
<h3>404 Not Found</h3>
|
||||
<p>Sorry, the page you are looking for is not found. Please try again later.</p>
|
||||
|
||||
<footer>Powered by TeaEdge.</footer>
|
||||
<footer>Powered by GoEdge.</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -9,7 +9,7 @@
|
||||
<h3>An error occurred.</h3>
|
||||
<p>Sorry, the page you are looking for is currently unavailable. Please try again later.</p>
|
||||
|
||||
<footer>Powered by TeaEdge.</footer>
|
||||
<footer>Powered by GoEdge.</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -9,7 +9,7 @@
|
||||
<h3>The website is shutdown.</h3>
|
||||
<p>Sorry, the page you are looking for is currently unavailable. Please try again later.</p>
|
||||
|
||||
<footer>Powered by TeaEdge.</footer>
|
||||
<footer>Powered by GoEdge.</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -9,7 +9,7 @@
|
||||
<h3>网站升级中</h3>
|
||||
<p>为了给您提供更好的服务,我们正在升级网站,请稍后重新访问。</p>
|
||||
|
||||
<footer>Powered by TeaEdge.</footer>
|
||||
<footer>Powered by GoEdge.</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -9,7 +9,7 @@
|
||||
<h3>网站暂时关闭</h3>
|
||||
<p>网站已被暂时关闭,请耐心等待我们的重新开通通知。</p>
|
||||
|
||||
<footer>Powered by TeaEdge.</footer>
|
||||
<footer>Powered by GoEdge.</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -5,19 +5,19 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/apps"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/nodes"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io/ioutil"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/gosock/pkg/gosock"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := apps.NewAppCmd().
|
||||
Version(teaconst.Version).
|
||||
Product(teaconst.ProductName).
|
||||
Usage(teaconst.ProcessName + " [-v|start|stop|restart|quit|test|service|daemon]")
|
||||
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|service|daemon|pprof]")
|
||||
|
||||
app.On("test", func() {
|
||||
err := nodes.NewNode().Test()
|
||||
@@ -37,25 +37,28 @@ func main() {
|
||||
fmt.Println("done")
|
||||
})
|
||||
app.On("quit", func() {
|
||||
pidFile := Tea.Root + "/bin/pid"
|
||||
data, err := ioutil.ReadFile(pidFile)
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
_, err := sock.Send(&gosock.Command{Code: "quit"})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]quit failed: " + err.Error())
|
||||
return
|
||||
}
|
||||
pid := types.Int(string(data))
|
||||
if pid == 0 {
|
||||
fmt.Println("[ERROR]quit failed: pid=0")
|
||||
return
|
||||
}
|
||||
fmt.Println("done")
|
||||
})
|
||||
app.On("pprof", func() {
|
||||
// TODO 自己指定端口
|
||||
addr := "127.0.0.1:6060"
|
||||
logs.Println("starting with pprof '" + addr + "'...")
|
||||
|
||||
process, err := os.FindProcess(pid)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if process != nil {
|
||||
_ = process.Signal(syscall.SIGQUIT)
|
||||
}
|
||||
go func() {
|
||||
err := http.ListenAndServe(addr, nil)
|
||||
if err != nil {
|
||||
logs.Println("[error]" + err.Error())
|
||||
}
|
||||
}()
|
||||
|
||||
node := nodes.NewNode()
|
||||
node.Start()
|
||||
})
|
||||
app.Run(func() {
|
||||
node := nodes.NewNode()
|
||||
|
||||
30
go.mod
30
go.mod
@@ -7,20 +7,32 @@ replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
|
||||
require (
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
|
||||
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
|
||||
github.com/andybalholm/brotli v1.0.3
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/chai2010/webp v1.1.0 // indirect
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
|
||||
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible
|
||||
github.com/golang/protobuf v1.4.2
|
||||
github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f
|
||||
github.com/iwind/gofcgi v0.0.0-20210506081859-17498ab3e9d7
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
|
||||
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
|
||||
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible
|
||||
github.com/mattn/go-sqlite3 v1.14.7
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible
|
||||
github.com/miekg/dns v1.1.43
|
||||
github.com/mssola/user_agent v0.5.2
|
||||
github.com/shirou/gopsutil v2.20.9+incompatible
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
|
||||
golang.org/x/text v0.3.2
|
||||
google.golang.org/grpc v1.32.0
|
||||
github.com/pires/go-proxyproto v0.6.1
|
||||
github.com/shirou/gopsutil v3.21.5+incompatible
|
||||
github.com/tklauser/go-sysconf v0.3.6 // indirect
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
|
||||
golang.org/x/text v0.3.6
|
||||
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
|
||||
google.golang.org/grpc v1.38.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
|
||||
)
|
||||
|
||||
115
go.sum
115
go.sum
@@ -7,22 +7,37 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
|
||||
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
|
||||
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
|
||||
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
|
||||
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49 h1:CtSi0QlA2Hy+nOh8JAZoiEBLW5pliAiKJ3l1Iq1472I=
|
||||
github.com/dop251/goja v0.0.0-20210804101310-32956a348b49/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
|
||||
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
@@ -30,6 +45,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
|
||||
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
|
||||
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
|
||||
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
|
||||
@@ -45,37 +62,48 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
|
||||
github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f h1:6Ws2H+eorfVUoMO2jta6A9nIdh8oi5/5LXo/LkAxR+E=
|
||||
github.com/iwind/TeaGo v0.0.0-20201020081413-7cf62d6f420f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
|
||||
github.com/iwind/gofcgi v0.0.0-20210506081859-17498ab3e9d7 h1:apv23QzWNmv0D76gB3+u/5kf0F/Yw4W8h489CWUZtss=
|
||||
github.com/iwind/gofcgi v0.0.0-20210506081859-17498ab3e9d7/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
|
||||
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060 h1:qdLtK4PDXxk2vMKkTWl5Fl9xqYuRCukzWAgJbLHdfOo=
|
||||
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
|
||||
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
|
||||
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
|
||||
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
|
||||
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
|
||||
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo=
|
||||
github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -86,17 +114,25 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
|
||||
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/shirou/gopsutil v2.20.9+incompatible h1:msXs2frUV+O/JLva9EDLpuJ84PrFsdCTCQex8PUdtkQ=
|
||||
github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
|
||||
github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
|
||||
github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
|
||||
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
|
||||
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@@ -104,13 +140,16 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -119,12 +158,17 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -136,34 +180,52 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
|
||||
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
|
||||
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
@@ -172,19 +234,24 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -2,8 +2,11 @@ package apps
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/iwind/gosock/pkg/gosock"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
@@ -11,7 +14,7 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// App命令帮助
|
||||
// AppCmd App命令帮助
|
||||
type AppCmd struct {
|
||||
product string
|
||||
version string
|
||||
@@ -20,10 +23,14 @@ type AppCmd struct {
|
||||
appendStrings []string
|
||||
|
||||
directives []*Directive
|
||||
|
||||
sock *gosock.Sock
|
||||
}
|
||||
|
||||
func NewAppCmd() *AppCmd {
|
||||
return &AppCmd{}
|
||||
return &AppCmd{
|
||||
sock: gosock.NewTmpSock(teaconst.ProcessName),
|
||||
}
|
||||
}
|
||||
|
||||
type CommandHelpOption struct {
|
||||
@@ -31,25 +38,25 @@ type CommandHelpOption struct {
|
||||
Description string
|
||||
}
|
||||
|
||||
// 产品
|
||||
// Product 产品
|
||||
func (this *AppCmd) Product(product string) *AppCmd {
|
||||
this.product = product
|
||||
return this
|
||||
}
|
||||
|
||||
// 版本
|
||||
// Version 版本
|
||||
func (this *AppCmd) Version(version string) *AppCmd {
|
||||
this.version = version
|
||||
return this
|
||||
}
|
||||
|
||||
// 使用方法
|
||||
// Usage 使用方法
|
||||
func (this *AppCmd) Usage(usage string) *AppCmd {
|
||||
this.usage = usage
|
||||
return this
|
||||
}
|
||||
|
||||
// 选项
|
||||
// Option 选项
|
||||
func (this *AppCmd) Option(code string, description string) *AppCmd {
|
||||
this.options = append(this.options, &CommandHelpOption{
|
||||
Code: code,
|
||||
@@ -58,13 +65,13 @@ func (this *AppCmd) Option(code string, description string) *AppCmd {
|
||||
return this
|
||||
}
|
||||
|
||||
// 附加内容
|
||||
// Append 附加内容
|
||||
func (this *AppCmd) Append(appendString string) *AppCmd {
|
||||
this.appendStrings = append(this.appendStrings, appendString)
|
||||
return this
|
||||
}
|
||||
|
||||
// 打印
|
||||
// Print 打印
|
||||
func (this *AppCmd) Print() {
|
||||
fmt.Println(this.product + " v" + this.version)
|
||||
|
||||
@@ -103,7 +110,7 @@ func (this *AppCmd) Print() {
|
||||
}
|
||||
}
|
||||
|
||||
// 添加指令
|
||||
// On 添加指令
|
||||
func (this *AppCmd) On(arg string, callback func()) {
|
||||
this.directives = append(this.directives, &Directive{
|
||||
Arg: arg,
|
||||
@@ -111,7 +118,7 @@ func (this *AppCmd) On(arg string, callback func()) {
|
||||
})
|
||||
}
|
||||
|
||||
// 运行
|
||||
// Run 运行
|
||||
func (this *AppCmd) Run(main func()) {
|
||||
// 获取参数
|
||||
args := os.Args[1:]
|
||||
@@ -161,7 +168,7 @@ func (this *AppCmd) Run(main func()) {
|
||||
|
||||
// 版本号
|
||||
func (this *AppCmd) runVersion() {
|
||||
fmt.Println(this.product+" v"+this.version, "(build: "+runtime.Version(), runtime.GOOS, runtime.GOARCH+")")
|
||||
fmt.Println(this.product+" v"+this.version, "(build: "+runtime.Version(), runtime.GOOS, runtime.GOARCH, teaconst.Tag+")")
|
||||
}
|
||||
|
||||
// 帮助
|
||||
@@ -171,12 +178,14 @@ func (this *AppCmd) runHelp() {
|
||||
|
||||
// 启动
|
||||
func (this *AppCmd) runStart() {
|
||||
proc := this.checkPid()
|
||||
if proc != nil {
|
||||
fmt.Println(this.product+" already started, pid:", proc.Pid)
|
||||
var pid = this.getPID()
|
||||
if pid > 0 {
|
||||
fmt.Println(this.product+" already started, pid:", pid)
|
||||
return
|
||||
}
|
||||
|
||||
_ = os.Setenv("EdgeBackground", "on")
|
||||
|
||||
cmd := exec.Command(os.Args[0])
|
||||
err := cmd.Start()
|
||||
if err != nil {
|
||||
@@ -189,18 +198,15 @@ func (this *AppCmd) runStart() {
|
||||
|
||||
// 停止
|
||||
func (this *AppCmd) runStop() {
|
||||
proc := this.checkPid()
|
||||
if proc == nil {
|
||||
var pid = this.getPID()
|
||||
if pid == 0 {
|
||||
fmt.Println(this.product + " not started yet")
|
||||
return
|
||||
}
|
||||
|
||||
// 停止进程
|
||||
_ = proc.Kill()
|
||||
_, _ = this.sock.Send(&gosock.Command{Code: "stop"})
|
||||
|
||||
// 在Windows上经常不能及时释放资源
|
||||
_ = DeletePid(Tea.Root + "/bin/pid")
|
||||
fmt.Println(this.product+" stopped ok, pid:", proc.Pid)
|
||||
fmt.Println(this.product+" stopped ok, pid:", types.String(pid))
|
||||
}
|
||||
|
||||
// 重启
|
||||
@@ -212,15 +218,24 @@ func (this *AppCmd) runRestart() {
|
||||
|
||||
// 状态
|
||||
func (this *AppCmd) runStatus() {
|
||||
proc := this.checkPid()
|
||||
if proc == nil {
|
||||
var pid = this.getPID()
|
||||
if pid == 0 {
|
||||
fmt.Println(this.product + " not started yet")
|
||||
} else {
|
||||
fmt.Println(this.product + " is running, pid: " + fmt.Sprintf("%d", proc.Pid))
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println(this.product + " is running, pid: " + types.String(pid))
|
||||
}
|
||||
|
||||
// 检查PID
|
||||
func (this *AppCmd) checkPid() *os.Process {
|
||||
return CheckPid(Tea.Root + "/bin/pid")
|
||||
// 获取当前的PID
|
||||
func (this *AppCmd) getPID() int {
|
||||
if !this.sock.IsListening() {
|
||||
return 0
|
||||
}
|
||||
|
||||
reply, err := this.sock.Send(&gosock.Command{Code: "pid"})
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return maps.NewMap(reply.Params).GetInt("pid")
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
// +build !windows
|
||||
|
||||
package apps
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// lock file
|
||||
func LockFile(fp *os.File) error {
|
||||
return syscall.Flock(int(fp.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
}
|
||||
|
||||
func UnlockFile(fp *os.File) error {
|
||||
return syscall.Flock(int(fp.Fd()), syscall.LOCK_UN)
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
// +build windows
|
||||
|
||||
package apps
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
// lock file
|
||||
func LockFile(fp *os.File) error {
|
||||
return errors.New("not implemented on windows")
|
||||
}
|
||||
|
||||
func UnlockFile(fp *os.File) error {
|
||||
return errors.New("not implemented on windows")
|
||||
}
|
||||
@@ -6,6 +6,10 @@ import (
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/utils/time"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type LogWriter struct {
|
||||
@@ -34,7 +38,24 @@ func (this *LogWriter) Init() {
|
||||
}
|
||||
|
||||
func (this *LogWriter) Write(message string) {
|
||||
log.Println(message)
|
||||
// 文件和行号
|
||||
var callDepth = 2
|
||||
var file string
|
||||
var line int
|
||||
var ok bool
|
||||
_, file, line, ok = runtime.Caller(callDepth)
|
||||
if ok {
|
||||
file = filepath.Base(file)
|
||||
}
|
||||
|
||||
backgroundEnv, _ := os.LookupEnv("EdgeBackground")
|
||||
if backgroundEnv != "on" {
|
||||
if len(file) > 0 {
|
||||
log.Println(message + " (" + file + ":" + strconv.Itoa(line) + ")")
|
||||
} else {
|
||||
log.Println(message)
|
||||
}
|
||||
}
|
||||
|
||||
if this.fileAppender != nil {
|
||||
_, err := this.fileAppender.AppendString(timeutil.Format("Y/m/d H:i:s ") + message + "\n")
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
var pidFileList = []*os.File{}
|
||||
|
||||
// 检查Pid
|
||||
func CheckPid(path string) *os.Process {
|
||||
// windows上打开的文件是不能删除的
|
||||
if runtime.GOOS == "windows" {
|
||||
if os.Remove(path) == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
// 是否能取得Lock
|
||||
err = LockFile(file)
|
||||
if err == nil {
|
||||
_ = UnlockFile(file)
|
||||
return nil
|
||||
}
|
||||
|
||||
pidBytes, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
pid := types.Int(string(pidBytes))
|
||||
|
||||
if pid <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
proc, _ := os.FindProcess(pid)
|
||||
return proc
|
||||
}
|
||||
|
||||
// 写入Pid
|
||||
func WritePid() error {
|
||||
path := Tea.Root + "/bin/pid"
|
||||
fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
events.On(events.EventQuit, func() {
|
||||
_ = fp.Close()
|
||||
})
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
err = LockFile(fp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pidFileList = append(pidFileList, fp) // hold the file pointers
|
||||
|
||||
_, err = fp.WriteString(fmt.Sprintf("%d", os.Getpid()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 写入Ppid
|
||||
func WritePpid(path string) error {
|
||||
fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
err = LockFile(fp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pidFileList = append(pidFileList, fp) // hold the file pointers
|
||||
|
||||
_, err = fp.WriteString(fmt.Sprintf("%d", os.Getppid()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 删除Pid
|
||||
func DeletePid(path string) error {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
for _, fp := range pidFileList {
|
||||
_ = UnlockFile(fp)
|
||||
_ = fp.Close()
|
||||
}
|
||||
return os.Remove(path)
|
||||
}
|
||||
41
internal/caches/errors.go
Normal file
41
internal/caches/errors.go
Normal file
@@ -0,0 +1,41 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import "errors"
|
||||
|
||||
// 常用的几个错误
|
||||
var (
|
||||
ErrNotFound = errors.New("cache not found")
|
||||
ErrFileIsWriting = errors.New("the file is writing")
|
||||
ErrInvalidRange = errors.New("invalid range")
|
||||
)
|
||||
|
||||
// CapacityError 容量错误
|
||||
// 独立出来是为了可以在有些场合下可以忽略,防止产生没必要的错误提示数量太多
|
||||
type CapacityError struct {
|
||||
err string
|
||||
}
|
||||
|
||||
func NewCapacityError(err string) error {
|
||||
return &CapacityError{err: err}
|
||||
}
|
||||
|
||||
func (this *CapacityError) Error() string {
|
||||
return this.err
|
||||
}
|
||||
|
||||
// CanIgnoreErr 检查错误是否可以忽略
|
||||
func CanIgnoreErr(err error) bool {
|
||||
if err == nil {
|
||||
return true
|
||||
}
|
||||
if err == ErrFileIsWriting {
|
||||
return true
|
||||
}
|
||||
_, ok := err.(*CapacityError)
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
16
internal/caches/errros_test.go
Normal file
16
internal/caches/errros_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCanIgnoreErr(t *testing.T) {
|
||||
a := assert.NewAssertion(t)
|
||||
|
||||
a.IsTrue(CanIgnoreErr(ErrFileIsWriting))
|
||||
a.IsTrue(CanIgnoreErr(NewCapacityError("over capcity")))
|
||||
a.IsFalse(CanIgnoreErr(ErrNotFound))
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package caches
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
)
|
||||
|
||||
type ItemType = int
|
||||
|
||||
@@ -16,14 +18,16 @@ type Item struct {
|
||||
HeaderSize int64 `json:"headerSize"`
|
||||
BodySize int64 `json:"bodySize"`
|
||||
MetaSize int64 `json:"metaSize"`
|
||||
Host string `json:"host"` // 主机名
|
||||
ServerId int64 `json:"serverId"` // 服务ID
|
||||
}
|
||||
|
||||
func (this *Item) IsExpired() bool {
|
||||
return this.ExpiredAt < time.Now().Unix()
|
||||
return this.ExpiredAt < utils.UnixTime()
|
||||
}
|
||||
|
||||
func (this *Item) TotalSize() int64 {
|
||||
return this.Size() + this.MetaSize + int64(len(this.Key)) + 64
|
||||
return this.Size() + this.MetaSize + int64(len(this.Key)) + int64(len(this.Host)) + 64
|
||||
}
|
||||
|
||||
func (this *Item) Size() int64 {
|
||||
|
||||
@@ -4,7 +4,13 @@ package caches
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
@@ -17,58 +23,78 @@ type FileList struct {
|
||||
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
|
||||
existsByHashStmt *sql.Stmt // 根据hash检查是否存在
|
||||
insertStmt *sql.Stmt // 写入数据
|
||||
selectByHashStmt *sql.Stmt // 使用hash查询数据
|
||||
deleteByHashStmt *sql.Stmt // 根据hash删除数据
|
||||
statStmt *sql.Stmt // 统计
|
||||
purgeStmt *sql.Stmt // 清理
|
||||
deleteAllStmt *sql.Stmt // 删除所有数据
|
||||
|
||||
oldTables []string
|
||||
itemsTableName string
|
||||
|
||||
isClosed bool
|
||||
|
||||
memoryCache *ttlcache.Cache
|
||||
}
|
||||
|
||||
func NewFileList(dir string) ListInterface {
|
||||
return &FileList{dir: dir}
|
||||
return &FileList{
|
||||
dir: dir,
|
||||
memoryCache: ttlcache.NewCache(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileList) Init() error {
|
||||
db, err := sql.Open("sqlite3", "file:"+this.dir+"/index.db?cache=shared&mode=rwc")
|
||||
// 检查目录是否存在
|
||||
_, err := os.Stat(this.dir)
|
||||
if err != nil {
|
||||
err = os.MkdirAll(this.dir, 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
remotelogs.Println("CACHE", "create cache dir '"+this.dir+"'")
|
||||
}
|
||||
|
||||
this.itemsTableName = "cacheItems_v2"
|
||||
|
||||
var dir = this.dir
|
||||
if dir == "/" {
|
||||
// 防止sqlite提示authority错误
|
||||
dir = ""
|
||||
}
|
||||
db, err := sql.Open("sqlite3", "file:"+dir+"/index.db?cache=shared&mode=rwc&_journal_mode=WAL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
_, err = db.Exec("VACUUM")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 创建
|
||||
// TODO accessesAt 用来存储访问时间,将来可以根据此访问时间删除不常访问的内容
|
||||
// 且访问时间只需要每隔一个小时存储一个整数值即可,因为不需要那么精确
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cacheItems" (
|
||||
"hash" varchar(32),
|
||||
"key" varchar(1024),
|
||||
"headerSize" integer DEFAULT 0,
|
||||
"bodySize" integer DEFAULT 0,
|
||||
"metaSize" integer DEFAULT 0,
|
||||
"expiredAt" integer DEFAULT 0,
|
||||
"accessedAt" integer DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
|
||||
ON "cacheItems" (
|
||||
"hash"
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS "expiredAt"
|
||||
ON "cacheItems" (
|
||||
"expiredAt"
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS "accessedAt"
|
||||
ON "cacheItems" (
|
||||
"accessedAt"
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.db = db
|
||||
|
||||
// 清除旧表
|
||||
this.oldTables = []string{
|
||||
"cacheItems",
|
||||
}
|
||||
err = this.removeOldTables()
|
||||
if err != nil {
|
||||
remotelogs.Warn("CACHE", "clean old tables failed: "+err.Error())
|
||||
}
|
||||
|
||||
// TODO 耗时过长,暂时不整理数据库
|
||||
/**_, err = db.Exec("VACUUM")
|
||||
if err != nil {
|
||||
return err
|
||||
}**/
|
||||
|
||||
// 创建
|
||||
err = this.initTables(db, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 读取总数量
|
||||
row := this.db.QueryRow("SELECT COUNT(*) FROM cacheItems")
|
||||
row := this.db.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
|
||||
if row.Err() != nil {
|
||||
return row.Err()
|
||||
}
|
||||
@@ -79,16 +105,56 @@ ON "cacheItems" (
|
||||
}
|
||||
this.total = total
|
||||
|
||||
// 常用语句
|
||||
this.existsByHashStmt, err = this.db.Prepare(`SELECT "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? AND expiredAt>? LIMIT 1`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.insertStmt, err = this.db.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.selectByHashStmt, err = this.db.Prepare(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? LIMIT 1`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.deleteByHashStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.statStmt, err = this.db.Prepare(`SELECT COUNT(*), IFNULL(SUM(headerSize+bodySize+metaSize), 0), IFNULL(SUM(headerSize+bodySize), 0) FROM "` + this.itemsTableName + `"`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.purgeStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" WHERE expiredAt<=? LIMIT ?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.deleteAllStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemsTableName + `"`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Reset() error {
|
||||
// 不错任何事情
|
||||
// 不做任何事情
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileList) Add(hash string, item *Item) error {
|
||||
_, err := this.db.Exec(`INSERT INTO cacheItems ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt") VALUES (?, ?, ?, ?, ?, ?)`, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt)
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.Host, item.ServerId, utils.UnixTime())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -102,54 +168,73 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
}
|
||||
|
||||
func (this *FileList) Exist(hash string) (bool, error) {
|
||||
row := this.db.QueryRow(`SELECT "bodySize" FROM cacheItems WHERE "hash"=? LIMIT 1`, hash)
|
||||
if row == nil {
|
||||
if this.isClosed {
|
||||
return false, nil
|
||||
}
|
||||
if row.Err() != nil {
|
||||
return false, row.Err()
|
||||
|
||||
item := this.memoryCache.Read(hash)
|
||||
if item != nil {
|
||||
return true, nil
|
||||
}
|
||||
var bodySize int
|
||||
err := row.Scan(&bodySize)
|
||||
|
||||
rows, err := this.existsByHashStmt.Query(hash, time.Now().Unix())
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// FindKeysWithPrefix 根据前缀进行查找
|
||||
func (this *FileList) FindKeysWithPrefix(prefix string) (keys []string, err error) {
|
||||
if len(prefix) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// TODO 需要优化上千万结果的情况
|
||||
|
||||
rows, err := this.db.Query(`SELECT "key" FROM cacheItems WHERE INSTR("key", ?)==1 LIMIT 100000`, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
|
||||
for rows.Next() {
|
||||
var key string
|
||||
err = rows.Scan(&key)
|
||||
if rows.Next() {
|
||||
var expiredAt int64
|
||||
err = rows.Scan(&expiredAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return false, nil
|
||||
}
|
||||
keys = append(keys, key)
|
||||
this.memoryCache.Write(hash, 1, expiredAt)
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// CleanPrefix 清理某个前缀的缓存数据
|
||||
func (this *FileList) CleanPrefix(prefix string) error {
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
return
|
||||
if len(prefix) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
defer func() {
|
||||
this.memoryCache.Clean()
|
||||
}()
|
||||
|
||||
var count = int64(10000)
|
||||
for {
|
||||
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0 WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+strconv.FormatInt(count, 10)+`)`, utils.UnixTime(), prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affectedRows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affectedRows < count {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileList) Remove(hash string) error {
|
||||
row := this.db.QueryRow(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt" FROM cacheItems WHERE "hash"=? LIMIT 1`, hash)
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 从缓存中删除
|
||||
this.memoryCache.Delete(hash)
|
||||
|
||||
row := this.selectByHashStmt.QueryRow(hash)
|
||||
if row.Err() != nil {
|
||||
return row.Err()
|
||||
}
|
||||
@@ -163,7 +248,7 @@ func (this *FileList) Remove(hash string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = this.db.Exec(`DELETE FROM cacheItems WHERE "hash"=?`, hash)
|
||||
_, err = this.deleteByHashStmt.Exec(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -181,27 +266,30 @@ func (this *FileList) Remove(hash string) error {
|
||||
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
|
||||
// callback 每次发现过期key的调用
|
||||
func (this *FileList) Purge(count int, callback func(hash string) error) error {
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
if count <= 0 {
|
||||
count = 1000
|
||||
}
|
||||
|
||||
rows, err := this.db.Query(`SELECT "hash" FROM cacheItems WHERE expiredAt<=? LIMIT ?`, time.Now().Unix(), count)
|
||||
rows, err := this.purgeStmt.Query(time.Now().Unix(), count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
|
||||
hashStrings := []string{}
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
hashStrings = append(hashStrings, hash)
|
||||
}
|
||||
_ = rows.Close() // 不能使用defer,防止读写冲突
|
||||
|
||||
// 不在 rows.Next() 循环中操作是为了避免死锁
|
||||
for _, hash := range hashStrings {
|
||||
@@ -220,7 +308,13 @@ func (this *FileList) Purge(count int, callback func(hash string) error) error {
|
||||
}
|
||||
|
||||
func (this *FileList) CleanAll() error {
|
||||
_, err := this.db.Exec("DELETE FROM cacheItems")
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.memoryCache.Clean()
|
||||
|
||||
_, err := this.deleteAllStmt.Exec()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -229,8 +323,12 @@ func (this *FileList) CleanAll() error {
|
||||
}
|
||||
|
||||
func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
if this.isClosed {
|
||||
return &Stat{}, nil
|
||||
}
|
||||
|
||||
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
|
||||
row := this.db.QueryRow("SELECT COUNT(*), IFNULL(SUM(headerSize+bodySize+metaSize), 0), IFNULL(SUM(headerSize+bodySize), 0) FROM cacheItems")
|
||||
row := this.statStmt.QueryRow()
|
||||
if row.Err() != nil {
|
||||
return nil, row.Err()
|
||||
}
|
||||
@@ -258,3 +356,101 @@ func (this *FileList) OnAdd(f func(item *Item)) {
|
||||
func (this *FileList) OnRemove(f func(item *Item)) {
|
||||
this.onRemove = f
|
||||
}
|
||||
|
||||
func (this *FileList) Close() error {
|
||||
this.isClosed = true
|
||||
|
||||
this.memoryCache.Destroy()
|
||||
|
||||
if this.db != nil {
|
||||
_ = this.existsByHashStmt.Close()
|
||||
_ = this.insertStmt.Close()
|
||||
_ = this.selectByHashStmt.Close()
|
||||
_ = this.deleteByHashStmt.Close()
|
||||
_ = this.statStmt.Close()
|
||||
_ = this.purgeStmt.Close()
|
||||
_ = this.deleteAllStmt.Close()
|
||||
|
||||
return this.db.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 初始化
|
||||
func (this *FileList) initTables(db *sql.DB, times int) error {
|
||||
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
|
||||
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
"hash" varchar(32),
|
||||
"key" varchar(1024),
|
||||
"headerSize" integer DEFAULT 0,
|
||||
"bodySize" integer DEFAULT 0,
|
||||
"metaSize" integer DEFAULT 0,
|
||||
"expiredAt" integer DEFAULT 0,
|
||||
"createdAt" integer DEFAULT 0,
|
||||
"host" varchar(128),
|
||||
"serverId" integer
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "createdAt"
|
||||
ON "` + this.itemsTableName + `" (
|
||||
"createdAt" ASC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "expiredAt"
|
||||
ON "` + this.itemsTableName + `" (
|
||||
"expiredAt" ASC
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
|
||||
ON "` + this.itemsTableName + `" (
|
||||
"hash" ASC
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "serverId"
|
||||
ON "` + this.itemsTableName + `" (
|
||||
"serverId" ASC
|
||||
);
|
||||
`)
|
||||
if err != nil {
|
||||
// 尝试删除重建
|
||||
if times < 3 {
|
||||
_, dropErr := db.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
|
||||
if dropErr == nil {
|
||||
return this.initTables(db, times+1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 删除过期不用的表格
|
||||
func (this *FileList) removeOldTables() error {
|
||||
rows, err := this.db.Query(`SELECT "name" FROM sqlite_master WHERE "type"='table'`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
for rows.Next() {
|
||||
var name string
|
||||
err = rows.Scan(&name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if lists.ContainsString(this.oldTables, name) {
|
||||
// 异步执行
|
||||
go func() {
|
||||
remotelogs.Println("CACHE", "remove old table '"+name+"' ...")
|
||||
_, _ = this.db.Exec(`DROP TABLE "` + name + `"`)
|
||||
remotelogs.Println("CACHE", "remove old table '"+name+"' done")
|
||||
}()
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,8 +4,10 @@ package caches
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -31,6 +33,8 @@ func TestFileList_Add(t *testing.T) {
|
||||
HeaderSize: 1,
|
||||
MetaSize: 2,
|
||||
BodySize: 3,
|
||||
Host: "teaos.cn",
|
||||
ServerId: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -44,11 +48,12 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 100_0000; i++ {
|
||||
u := "http://edge.teaos.cn/123456" + strconv.Itoa(i)
|
||||
err = list.Add(stringutil.Md5(u), &Item{
|
||||
before := time.Now()
|
||||
for i := 0; i < 2000_0000; i++ {
|
||||
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
|
||||
_ = list.Add(stringutil.Md5(u), &Item{
|
||||
Key: u,
|
||||
ExpiredAt: time.Now().Unix(),
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1,
|
||||
MetaSize: 2,
|
||||
BodySize: 3,
|
||||
@@ -56,6 +61,10 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i > 0 && i%10_000 == 0 {
|
||||
t.Log(i, int(10000/time.Since(before).Seconds()), "qps")
|
||||
before = time.Now()
|
||||
}
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
@@ -66,6 +75,10 @@ func TestFileList_Exist(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := time.Now()
|
||||
defer func() {
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
{
|
||||
exists, err := list.Exist(stringutil.Md5("123456"))
|
||||
if err != nil {
|
||||
@@ -74,7 +87,7 @@ func TestFileList_Exist(t *testing.T) {
|
||||
t.Log("exists:", exists)
|
||||
}
|
||||
{
|
||||
exists, err := list.Exist(stringutil.Md5("654321"))
|
||||
exists, err := list.Exist(stringutil.Md5("http://edge.teaos.cn/1234561"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -82,18 +95,70 @@ func TestFileList_Exist(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileList_FindKeysWithPrefix(t *testing.T) {
|
||||
func TestFileList_Exist_Many_DB(t *testing.T) {
|
||||
// 测试在多个数据库下的性能
|
||||
var listSlice = []ListInterface{}
|
||||
for i := 1; i <= 10; i++ {
|
||||
list := NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listSlice = append(listSlice, list)
|
||||
}
|
||||
|
||||
var wg = sync.WaitGroup{}
|
||||
var threads = 8
|
||||
wg.Add(threads)
|
||||
|
||||
var count = 200_000
|
||||
var countLocker sync.Mutex
|
||||
var tasks = make(chan int, count)
|
||||
for i := 0; i < count; i++ {
|
||||
tasks <- i
|
||||
}
|
||||
|
||||
var hash = stringutil.Md5("http://edge.teaos.cn/1234561")
|
||||
|
||||
before := time.Now()
|
||||
defer func() {
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
for i := 0; i < threads; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-tasks:
|
||||
countLocker.Lock()
|
||||
count--
|
||||
countLocker.Unlock()
|
||||
|
||||
var list = listSlice[rands.Int(0, len(listSlice)-1)]
|
||||
_, _ = list.Exist(hash)
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
t.Log("left:", count)
|
||||
}
|
||||
|
||||
func TestFileList_CleanPrefix(t *testing.T) {
|
||||
list := NewFileList(Tea.Root + "/data")
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := time.Now()
|
||||
keys, err := list.FindKeysWithPrefix("1234")
|
||||
err = list.CleanPrefix("1234")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("keys:", keys)
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
@@ -170,3 +235,35 @@ func TestFileList_CleanAll(t *testing.T) {
|
||||
t.Log("ok")
|
||||
t.Log(list.Count())
|
||||
}
|
||||
|
||||
func TestFileList_Conflict(t *testing.T) {
|
||||
list := NewFileList(Tea.Root + "/data").(*FileList)
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rows, err := list.purgeStmt.Query(time.Now().Unix(), 1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(5 * time.Second)
|
||||
_ = rows.Close()
|
||||
}()
|
||||
|
||||
t.Log("before exists")
|
||||
t.Log(list.Exist("123456"))
|
||||
t.Log("after exists")
|
||||
}
|
||||
|
||||
func BenchmarkFileList_Exist(b *testing.B) {
|
||||
list := NewFileList(Tea.Root + "/data")
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,23 +3,31 @@
|
||||
package caches
|
||||
|
||||
type ListInterface interface {
|
||||
// Init 初始化
|
||||
Init() error
|
||||
|
||||
// Reset 重置数据
|
||||
Reset() error
|
||||
|
||||
// Add 添加内容
|
||||
Add(hash string, item *Item) error
|
||||
|
||||
// Exist 检查内容是否存在
|
||||
Exist(hash string) (bool, error)
|
||||
|
||||
// FindKeysWithPrefix 根据前缀进行查找
|
||||
FindKeysWithPrefix(prefix string) (keys []string, err error)
|
||||
// CleanPrefix 清除某个前缀的缓存
|
||||
CleanPrefix(prefix string) error
|
||||
|
||||
// Remove 删除内容
|
||||
Remove(hash string) error
|
||||
|
||||
// Purge 清理过期数据
|
||||
Purge(count int, callback func(hash string) error) error
|
||||
|
||||
// CleanAll 清除所有缓存
|
||||
CleanAll() error
|
||||
|
||||
// Stat 统计
|
||||
Stat(check func(hash string) bool) (*Stat, error)
|
||||
|
||||
// Count 总数量
|
||||
@@ -30,4 +38,7 @@ type ListInterface interface {
|
||||
|
||||
// OnRemove 删除事件
|
||||
OnRemove(f func(item *Item))
|
||||
|
||||
// Close 关闭
|
||||
Close() error
|
||||
}
|
||||
@@ -1,42 +1,76 @@
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// MemoryList 内存缓存列表管理
|
||||
type MemoryList struct {
|
||||
m map[string]*Item // hash => item
|
||||
itemMaps map[string]map[string]*Item // prefix => { hash => item }
|
||||
prefixes []string
|
||||
locker sync.RWMutex
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
|
||||
purgeIndex int
|
||||
}
|
||||
|
||||
func NewMemoryList() ListInterface {
|
||||
return &MemoryList{
|
||||
m: map[string]*Item{},
|
||||
itemMaps: map[string]map[string]*Item{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *MemoryList) Init() error {
|
||||
// 内存列表不需要初始化
|
||||
this.prefixes = []string{"000"}
|
||||
for i := 100; i <= 999; i++ {
|
||||
this.prefixes = append(this.prefixes, strconv.Itoa(i))
|
||||
}
|
||||
|
||||
for _, prefix := range this.prefixes {
|
||||
this.itemMaps[prefix] = map[string]*Item{}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *MemoryList) Reset() error {
|
||||
this.locker.Lock()
|
||||
this.m = map[string]*Item{}
|
||||
for key := range this.itemMaps {
|
||||
this.itemMaps[key] = map[string]*Item{}
|
||||
}
|
||||
this.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *MemoryList) Add(hash string, item *Item) error {
|
||||
this.locker.Lock()
|
||||
|
||||
prefix := this.prefix(hash)
|
||||
itemMap, ok := this.itemMaps[prefix]
|
||||
if !ok {
|
||||
itemMap = map[string]*Item{}
|
||||
this.itemMaps[prefix] = itemMap
|
||||
}
|
||||
|
||||
// 先删除,为了可以正确触发统计
|
||||
oldItem, ok := itemMap[hash]
|
||||
if ok {
|
||||
if this.onRemove != nil {
|
||||
this.onRemove(oldItem)
|
||||
}
|
||||
}
|
||||
|
||||
// 添加
|
||||
if this.onAdd != nil {
|
||||
this.onAdd(item)
|
||||
}
|
||||
this.m[hash] = item
|
||||
|
||||
itemMap[hash] = item
|
||||
this.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -45,7 +79,12 @@ func (this *MemoryList) Exist(hash string) (bool, error) {
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
item, ok := this.m[hash]
|
||||
prefix := this.prefix(hash)
|
||||
itemMap, ok := this.itemMaps[prefix]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
item, ok := itemMap[hash]
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
@@ -53,29 +92,37 @@ func (this *MemoryList) Exist(hash string) (bool, error) {
|
||||
return !item.IsExpired(), nil
|
||||
}
|
||||
|
||||
// FindKeysWithPrefix 根据前缀进行查找
|
||||
func (this *MemoryList) FindKeysWithPrefix(prefix string) (keys []string, err error) {
|
||||
// CleanPrefix 根据前缀进行清除
|
||||
func (this *MemoryList) CleanPrefix(prefix string) error {
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
// TODO 需要优化性能,支持千万级数据低于1s的处理速度
|
||||
for _, item := range this.m {
|
||||
if strings.HasPrefix(item.Key, prefix) {
|
||||
keys = append(keys, item.Key)
|
||||
for _, itemMap := range this.itemMaps {
|
||||
for _, item := range itemMap {
|
||||
if strings.HasPrefix(item.Key, prefix) {
|
||||
item.ExpiredAt = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *MemoryList) Remove(hash string) error {
|
||||
this.locker.Lock()
|
||||
|
||||
item, ok := this.m[hash]
|
||||
itemMap, ok := this.itemMaps[this.prefix(hash)]
|
||||
if !ok {
|
||||
this.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
item, ok := itemMap[hash]
|
||||
if ok {
|
||||
if this.onRemove != nil {
|
||||
this.onRemove(item)
|
||||
}
|
||||
delete(this.m, hash)
|
||||
delete(itemMap, hash)
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
@@ -88,7 +135,20 @@ func (this *MemoryList) Remove(hash string) error {
|
||||
func (this *MemoryList) Purge(count int, callback func(hash string) error) error {
|
||||
this.locker.Lock()
|
||||
deletedHashList := []string{}
|
||||
for hash, item := range this.m {
|
||||
|
||||
if this.purgeIndex >= len(this.prefixes) {
|
||||
this.purgeIndex = 0
|
||||
}
|
||||
prefix := this.prefixes[this.purgeIndex]
|
||||
|
||||
this.purgeIndex++
|
||||
|
||||
itemMap, ok := this.itemMaps[prefix]
|
||||
if !ok {
|
||||
this.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
for hash, item := range itemMap {
|
||||
if count <= 0 {
|
||||
break
|
||||
}
|
||||
@@ -97,7 +157,7 @@ func (this *MemoryList) Purge(count int, callback func(hash string) error) error
|
||||
if this.onRemove != nil {
|
||||
this.onRemove(item)
|
||||
}
|
||||
delete(this.m, hash)
|
||||
delete(itemMap, hash)
|
||||
deletedHashList = append(deletedHashList, hash)
|
||||
}
|
||||
|
||||
@@ -129,13 +189,15 @@ func (this *MemoryList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
Count: 0,
|
||||
Size: 0,
|
||||
}
|
||||
for hash, item := range this.m {
|
||||
if !item.IsExpired() {
|
||||
// 检查文件是否存在、内容是否正确等
|
||||
if check != nil && check(hash) {
|
||||
result.Count++
|
||||
result.ValueSize += item.Size()
|
||||
result.Size += item.TotalSize()
|
||||
for _, itemMap := range this.itemMaps {
|
||||
for hash, item := range itemMap {
|
||||
if !item.IsExpired() {
|
||||
// 检查文件是否存在、内容是否正确等
|
||||
if check != nil && check(hash) {
|
||||
result.Count++
|
||||
result.ValueSize += item.Size()
|
||||
result.Size += item.TotalSize()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -145,9 +207,12 @@ func (this *MemoryList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
// Count 总数量
|
||||
func (this *MemoryList) Count() (int64, error) {
|
||||
this.locker.RLock()
|
||||
count := int64(len(this.m))
|
||||
var count = 0
|
||||
for _, itemMap := range this.itemMaps {
|
||||
count += len(itemMap)
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
return count, nil
|
||||
return int64(count), nil
|
||||
}
|
||||
|
||||
// OnAdd 添加事件
|
||||
@@ -159,3 +224,31 @@ func (this *MemoryList) OnAdd(f func(item *Item)) {
|
||||
func (this *MemoryList) OnRemove(f func(item *Item)) {
|
||||
this.onRemove = f
|
||||
}
|
||||
|
||||
func (this *MemoryList) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *MemoryList) print(t *testing.T) {
|
||||
this.locker.Lock()
|
||||
for _, itemMap := range this.itemMaps {
|
||||
if len(itemMap) > 0 {
|
||||
logs.PrintAsJSON(itemMap, t)
|
||||
}
|
||||
}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *MemoryList) prefix(hash string) string {
|
||||
var prefix string
|
||||
if len(hash) > 3 {
|
||||
prefix = hash[:3]
|
||||
} else {
|
||||
prefix = hash
|
||||
}
|
||||
_, ok := this.itemMaps[prefix]
|
||||
if !ok {
|
||||
prefix = "000"
|
||||
}
|
||||
return prefix
|
||||
}
|
||||
|
||||
@@ -3,14 +3,17 @@ package caches
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestList_Add(t *testing.T) {
|
||||
list := &MemoryList{}
|
||||
func TestMemoryList_Add(t *testing.T) {
|
||||
list := NewMemoryList().(*MemoryList)
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
@@ -21,11 +24,18 @@ func TestList_Add(t *testing.T) {
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
t.Log(list.m)
|
||||
_ = list.Add("123456", &Item{
|
||||
Key: "c1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
t.Log(list.prefixes)
|
||||
logs.PrintAsJSON(list.itemMaps, t)
|
||||
}
|
||||
|
||||
func TestList_Remove(t *testing.T) {
|
||||
list := &MemoryList{}
|
||||
func TestMemoryList_Remove(t *testing.T) {
|
||||
list := NewMemoryList().(*MemoryList)
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
@@ -37,11 +47,12 @@ func TestList_Remove(t *testing.T) {
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
_ = list.Remove("b")
|
||||
t.Log(list.m)
|
||||
list.print(t)
|
||||
}
|
||||
|
||||
func TestList_Purge(t *testing.T) {
|
||||
list := &MemoryList{}
|
||||
func TestMemoryList_Purge(t *testing.T) {
|
||||
list := NewMemoryList().(*MemoryList)
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
@@ -66,11 +77,35 @@ func TestList_Purge(t *testing.T) {
|
||||
t.Log("delete:", hash)
|
||||
return nil
|
||||
})
|
||||
t.Log(list.m)
|
||||
list.print(t)
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
_ = list.Purge(100, func(hash string) error {
|
||||
t.Log("delete:", hash)
|
||||
return nil
|
||||
})
|
||||
t.Log(list.purgeIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestList_Stat(t *testing.T) {
|
||||
list := &MemoryList{}
|
||||
func TestMemoryList_Purge_Large_List(t *testing.T) {
|
||||
list := NewMemoryList().(*MemoryList)
|
||||
_ = list.Init()
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
_ = list.Add("a"+strconv.Itoa(i), &Item{
|
||||
Key: "a" + strconv.Itoa(i),
|
||||
ExpiredAt: time.Now().Unix() + int64(rands.Int(0, 24*3600)),
|
||||
HeaderSize: 1024,
|
||||
})
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Hour)
|
||||
}
|
||||
|
||||
func TestMemoryList_Stat(t *testing.T) {
|
||||
list := NewMemoryList()
|
||||
_ = list.Init()
|
||||
_ = list.Add("a", &Item{
|
||||
Key: "a1",
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
@@ -99,9 +134,37 @@ func TestList_Stat(t *testing.T) {
|
||||
t.Log(result)
|
||||
}
|
||||
|
||||
func TestList_FindKeysWithPrefix(t *testing.T) {
|
||||
list := &MemoryList{}
|
||||
func TestMemoryList_CleanPrefix(t *testing.T) {
|
||||
list := NewMemoryList()
|
||||
_ = list.Init()
|
||||
before := time.Now()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
key := "http://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
|
||||
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
|
||||
Key: key,
|
||||
ExpiredAt: time.Now().Unix() + 3600,
|
||||
BodySize: 0,
|
||||
HeaderSize: 0,
|
||||
})
|
||||
}
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
|
||||
before = time.Now()
|
||||
err := list.CleanPrefix("http://www.teaos.cn/hello/10")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
logs.Println(list.Stat(func(hash string) bool {
|
||||
return true
|
||||
}))
|
||||
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
func TestMemoryList_GC(t *testing.T) {
|
||||
list := NewMemoryList().(*MemoryList)
|
||||
_ = list.Init()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
key := "http://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
|
||||
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
|
||||
@@ -111,13 +174,17 @@ func TestList_FindKeysWithPrefix(t *testing.T) {
|
||||
HeaderSize: 0,
|
||||
})
|
||||
}
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
time.Sleep(10 * time.Second)
|
||||
t.Log("clean...", len(list.itemMaps))
|
||||
_ = list.CleanAll()
|
||||
t.Log("cleanAll...", len(list.itemMaps))
|
||||
before := time.Now()
|
||||
//runtime.GC()
|
||||
t.Log("gc cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
|
||||
before = time.Now()
|
||||
keys, err := list.FindKeysWithPrefix("http://www.teaos.cn/hello/5000")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(len(keys))
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
timeout := time.NewTimer(2 * time.Minute)
|
||||
<-timeout.C
|
||||
t.Log("2 minutes passed")
|
||||
|
||||
time.Sleep(30 * time.Minute)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
@@ -162,3 +163,25 @@ func (this *Manager) TotalMemorySize() int64 {
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// FindAllCachePaths 所有缓存路径
|
||||
func (this *Manager) FindAllCachePaths() []string {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var result = []string{}
|
||||
for _, policy := range this.policyMap {
|
||||
if policy.Type == serverconfigs.CachePolicyStorageFile {
|
||||
if policy.Options != nil {
|
||||
dir, ok := policy.Options["dir"]
|
||||
if ok {
|
||||
var dirString = types.String(dir)
|
||||
if len(dirString) > 0 {
|
||||
result = append(result, dirString)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -12,6 +12,9 @@ type Reader interface {
|
||||
// Status 状态码
|
||||
Status() int
|
||||
|
||||
// LastModified 最后修改时间
|
||||
LastModified() int64
|
||||
|
||||
// ReadHeader 读取Header
|
||||
ReadHeader(buf []byte, callback ReaderFunc) error
|
||||
|
||||
|
||||
@@ -34,13 +34,7 @@ func (this *FileReader) Init() error {
|
||||
}
|
||||
}()
|
||||
|
||||
// 读取状态
|
||||
_, err := this.fp.Seek(SizeExpiresAt, io.SeekStart)
|
||||
if err != nil {
|
||||
_ = this.discard()
|
||||
return err
|
||||
}
|
||||
buf := make([]byte, 3)
|
||||
var buf = make([]byte, SizeMeta)
|
||||
ok, err := this.readToBuff(this.fp, buf)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -48,37 +42,18 @@ func (this *FileReader) Init() error {
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
status := types.Int(string(buf))
|
||||
|
||||
status := types.Int(string(buf[SizeExpiresAt : SizeExpiresAt+SizeStatus]))
|
||||
if status < 100 || status > 999 {
|
||||
return errors.New("invalid status")
|
||||
}
|
||||
this.status = status
|
||||
|
||||
// URL
|
||||
_, err = this.fp.Seek(SizeExpiresAt+SizeStatus, io.SeekStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bytes4 := make([]byte, 4)
|
||||
ok, err = this.readToBuff(this.fp, bytes4)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
urlLength := binary.BigEndian.Uint32(bytes4)
|
||||
urlLength := binary.BigEndian.Uint32(buf[SizeExpiresAt+SizeStatus : SizeExpiresAt+SizeStatus+SizeURLLength])
|
||||
|
||||
// header
|
||||
ok, err = this.readToBuff(this.fp, bytes4)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
headerSize := int(binary.BigEndian.Uint32(bytes4))
|
||||
headerSize := int(binary.BigEndian.Uint32(buf[SizeExpiresAt+SizeStatus+SizeURLLength : SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength]))
|
||||
if headerSize == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -86,16 +61,9 @@ func (this *FileReader) Init() error {
|
||||
this.headerOffset = int64(SizeMeta) + int64(urlLength)
|
||||
|
||||
// body
|
||||
bytes8 := make([]byte, 8)
|
||||
ok, err = this.readToBuff(this.fp, bytes8)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
bodySize := int(binary.BigEndian.Uint64(bytes8))
|
||||
bodySize := int(binary.BigEndian.Uint64(buf[SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength : SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength+SizeBodyLength]))
|
||||
if bodySize == 0 {
|
||||
isOk = true
|
||||
return nil
|
||||
}
|
||||
this.bodySize = int64(bodySize)
|
||||
@@ -114,6 +82,14 @@ func (this *FileReader) Status() int {
|
||||
return this.status
|
||||
}
|
||||
|
||||
func (this *FileReader) LastModified() int64 {
|
||||
stat, err := this.fp.Stat()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return stat.ModTime().Unix()
|
||||
}
|
||||
|
||||
func (this *FileReader) HeaderSize() int64 {
|
||||
return int64(this.headerSize)
|
||||
}
|
||||
|
||||
@@ -49,6 +49,9 @@ func TestFileReader(t *testing.T) {
|
||||
t.Log("body:", string(buf[:n]))
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileReader_Range(t *testing.T) {
|
||||
|
||||
@@ -24,6 +24,10 @@ func (this *MemoryReader) Status() int {
|
||||
return this.item.Status
|
||||
}
|
||||
|
||||
func (this *MemoryReader) LastModified() int64 {
|
||||
return this.item.ModifiedAt
|
||||
}
|
||||
|
||||
func (this *MemoryReader) HeaderSize() int64 {
|
||||
return int64(len(this.item.HeaderValue))
|
||||
}
|
||||
|
||||
@@ -36,12 +36,6 @@ const (
|
||||
SizeMeta = SizeExpiresAt + SizeStatus + SizeURLLength + SizeHeaderLength + SizeBodyLength
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("cache not found")
|
||||
ErrFileIsWriting = errors.New("the file is writing")
|
||||
ErrInvalidRange = errors.New("invalid range")
|
||||
)
|
||||
|
||||
// FileStorage 文件缓存
|
||||
// 文件结构:
|
||||
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
|
||||
@@ -51,14 +45,16 @@ type FileStorage struct {
|
||||
memoryStorage *MemoryStorage // 一级缓存
|
||||
totalSize int64
|
||||
|
||||
list ListInterface
|
||||
locker sync.RWMutex
|
||||
ticker *utils.Ticker
|
||||
list ListInterface
|
||||
writingKeyMap map[string]bool // key => bool
|
||||
locker sync.RWMutex
|
||||
ticker *utils.Ticker
|
||||
}
|
||||
|
||||
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
|
||||
return &FileStorage{
|
||||
policy: policy,
|
||||
policy: policy,
|
||||
writingKeyMap: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,9 +191,10 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
|
||||
}
|
||||
}
|
||||
|
||||
_, path := this.keyPath(key)
|
||||
hash, path := this.keyPath(key)
|
||||
|
||||
// TODO 尝试使用mmap加快读取速度
|
||||
var isOk = false
|
||||
fp, err := os.OpenFile(path, os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
@@ -205,6 +202,21 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
defer func() {
|
||||
if !isOk {
|
||||
_ = fp.Close()
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
|
||||
// 检查文件记录是否已过期
|
||||
exists, err := this.list.Exist(hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
reader := NewFileReader(fp)
|
||||
if err != nil {
|
||||
@@ -214,6 +226,8 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isOk = true
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
@@ -227,17 +241,36 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
|
||||
}
|
||||
}
|
||||
|
||||
// 是否正在写入
|
||||
var isWriting = false
|
||||
this.locker.Lock()
|
||||
_, ok := this.writingKeyMap[key]
|
||||
this.locker.Unlock()
|
||||
if ok {
|
||||
return nil, ErrFileIsWriting
|
||||
}
|
||||
this.locker.Lock()
|
||||
this.writingKeyMap[key] = true
|
||||
this.locker.Unlock()
|
||||
defer func() {
|
||||
if !isWriting {
|
||||
this.locker.Lock()
|
||||
delete(this.writingKeyMap, key)
|
||||
this.locker.Unlock()
|
||||
}
|
||||
}()
|
||||
|
||||
// 检查是否超出最大值
|
||||
count, err := this.list.Count()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if this.policy.MaxKeys > 0 && count > this.policy.MaxKeys {
|
||||
return nil, errors.New("write file cache failed: too many keys in cache storage")
|
||||
return nil, NewCapacityError("write file cache failed: too many keys in cache storage")
|
||||
}
|
||||
capacityBytes := this.diskCapacityBytes()
|
||||
if capacityBytes > 0 && capacityBytes <= this.totalSize {
|
||||
return nil, errors.New("write file cache failed: over disk size, current total size: " + strconv.FormatInt(this.totalSize, 10) + " bytes, capacity: " + strconv.FormatInt(capacityBytes, 10))
|
||||
return nil, NewCapacityError("write file cache failed: over disk size, current total size: " + strconv.FormatInt(this.totalSize, 10) + " bytes, capacity: " + strconv.FormatInt(capacityBytes, 10))
|
||||
}
|
||||
|
||||
hash := stringutil.Md5(key)
|
||||
@@ -264,6 +297,7 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isWriting = true
|
||||
|
||||
isOk := false
|
||||
removeOnFailure := true
|
||||
@@ -348,7 +382,11 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int) (Wr
|
||||
|
||||
isOk = true
|
||||
|
||||
return NewFileWriter(writer, key, expiredAt), nil
|
||||
return NewFileWriter(writer, key, expiredAt, func() {
|
||||
this.locker.Lock()
|
||||
delete(this.writingKeyMap, key)
|
||||
this.locker.Unlock()
|
||||
}), nil
|
||||
}
|
||||
|
||||
// AddToList 添加到List
|
||||
@@ -435,6 +473,7 @@ func (this *FileStorage) CleanAll() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 改成待删除
|
||||
subDirs, err := fp.Readdir(-1)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -451,13 +490,22 @@ func (this *FileStorage) CleanAll() error {
|
||||
continue
|
||||
}
|
||||
|
||||
// 删除目录
|
||||
err = os.RemoveAll(dir + "/" + subDir)
|
||||
// 修改目录名
|
||||
tmpDir := dir + "/" + subDir + "-deleted"
|
||||
err = os.Rename(dir+"/"+subDir, tmpDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 重新遍历待删除
|
||||
go func() {
|
||||
err = this.cleanDeletedDirs(dir)
|
||||
if err != nil {
|
||||
remotelogs.Warn("CACHE", "delete '*-deleted' dirs failed: "+err.Error())
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -473,33 +521,18 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
|
||||
|
||||
// 目录
|
||||
if urlType == "dir" {
|
||||
resultKeys := []string{}
|
||||
for _, key := range keys {
|
||||
subKeys, err := this.list.FindKeysWithPrefix(key)
|
||||
err := this.list.CleanPrefix(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resultKeys = append(resultKeys, subKeys...)
|
||||
}
|
||||
keys = resultKeys
|
||||
}
|
||||
|
||||
// 文件
|
||||
for _, key := range keys {
|
||||
hash, path := this.keyPath(key)
|
||||
exists, err := this.list.Exist(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
err = os.Remove(path)
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
@@ -525,6 +558,8 @@ func (this *FileStorage) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
|
||||
_ = this.list.Close()
|
||||
}
|
||||
|
||||
// TotalDiskSize 消耗的磁盘尺寸
|
||||
@@ -695,7 +730,7 @@ func (this *FileStorage) decodeFile(path string) (*Item, error) {
|
||||
|
||||
// 清理任务
|
||||
func (this *FileStorage) purgeLoop() {
|
||||
_ = this.list.Purge(1000, func(hash string) error {
|
||||
err := this.list.Purge(1000, func(hash string) error {
|
||||
path := this.hashPath(hash)
|
||||
err := os.Remove(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
@@ -703,6 +738,9 @@ func (this *FileStorage) purgeLoop() {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
remotelogs.Warn("CACHE", "purge file storage failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileStorage) readToBuff(fp *os.File, buf []byte) (ok bool, err error) {
|
||||
@@ -743,3 +781,34 @@ func (this *FileStorage) diskCapacityBytes() int64 {
|
||||
}
|
||||
return c1
|
||||
}
|
||||
|
||||
// 清理 *-deleted 目录
|
||||
// 由于在很多硬盘上耗时非常久,所以应该放在后台运行
|
||||
func (this *FileStorage) cleanDeletedDirs(dir string) error {
|
||||
fp, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = fp.Close()
|
||||
}()
|
||||
subDirs, err := fp.Readdir(-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, info := range subDirs {
|
||||
subDir := info.Name()
|
||||
if !strings.HasSuffix(subDir, "-deleted") {
|
||||
continue
|
||||
}
|
||||
|
||||
// 删除
|
||||
err = os.RemoveAll(dir + "/" + subDir)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package caches
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/cespare/xxhash"
|
||||
"strconv"
|
||||
@@ -18,6 +18,11 @@ type MemoryItem struct {
|
||||
BodyValue []byte
|
||||
Status int
|
||||
IsDone bool
|
||||
ModifiedAt int64
|
||||
}
|
||||
|
||||
func (this *MemoryItem) IsExpired() bool {
|
||||
return this.ExpiredAt < utils.UnixTime()
|
||||
}
|
||||
|
||||
type MemoryStorage struct {
|
||||
@@ -28,19 +33,23 @@ type MemoryStorage struct {
|
||||
ticker *utils.Ticker
|
||||
purgeDuration time.Duration
|
||||
totalSize int64
|
||||
writingKeyMap map[string]bool // key => bool
|
||||
}
|
||||
|
||||
func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy) *MemoryStorage {
|
||||
return &MemoryStorage{
|
||||
policy: policy,
|
||||
list: NewMemoryList(),
|
||||
locker: &sync.RWMutex{},
|
||||
valuesMap: map[uint64]*MemoryItem{},
|
||||
policy: policy,
|
||||
list: NewMemoryList(),
|
||||
locker: &sync.RWMutex{},
|
||||
valuesMap: map[uint64]*MemoryItem{},
|
||||
writingKeyMap: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// Init 初始化
|
||||
func (this *MemoryStorage) Init() error {
|
||||
_ = this.list.Init()
|
||||
|
||||
this.list.OnAdd(func(item *Item) {
|
||||
atomic.AddInt64(&this.totalSize, item.TotalSize())
|
||||
})
|
||||
@@ -49,7 +58,7 @@ func (this *MemoryStorage) Init() error {
|
||||
})
|
||||
|
||||
if this.purgeDuration <= 0 {
|
||||
this.purgeDuration = 30 * time.Second
|
||||
this.purgeDuration = 10 * time.Second
|
||||
}
|
||||
|
||||
// 启动定时清理任务
|
||||
@@ -68,10 +77,9 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
|
||||
hash := this.hash(key)
|
||||
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
item := this.valuesMap[hash]
|
||||
if item == nil || !item.IsDone {
|
||||
this.locker.RUnlock()
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
@@ -79,10 +87,13 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
|
||||
reader := NewMemoryReader(item)
|
||||
err := reader.Init()
|
||||
if err != nil {
|
||||
this.locker.RUnlock()
|
||||
return nil, err
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
return reader, nil
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
|
||||
_ = this.Delete(key)
|
||||
|
||||
@@ -91,26 +102,54 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
|
||||
|
||||
// OpenWriter 打开缓存写入器等待写入
|
||||
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int) (Writer, error) {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
// 是否正在写入
|
||||
var isWriting = false
|
||||
_, ok := this.writingKeyMap[key]
|
||||
if ok {
|
||||
return nil, ErrFileIsWriting
|
||||
}
|
||||
this.writingKeyMap[key] = true
|
||||
defer func() {
|
||||
if !isWriting {
|
||||
delete(this.writingKeyMap, key)
|
||||
}
|
||||
}()
|
||||
|
||||
// 检查是否过期
|
||||
hash := this.hash(key)
|
||||
item, ok := this.valuesMap[hash]
|
||||
if ok && !item.IsExpired() {
|
||||
return nil, ErrFileIsWriting
|
||||
}
|
||||
|
||||
// 检查是否超出最大值
|
||||
totalKeys, err := this.list.Count()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if this.policy.MaxKeys > 0 && totalKeys > this.policy.MaxKeys {
|
||||
return nil, errors.New("write memory cache failed: too many keys in cache storage")
|
||||
return nil, NewCapacityError("write memory cache failed: too many keys in cache storage")
|
||||
}
|
||||
capacityBytes := this.memoryCapacityBytes()
|
||||
if capacityBytes > 0 && capacityBytes <= this.totalSize {
|
||||
return nil, errors.New("write memory cache failed: over memory size, real size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
|
||||
return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
|
||||
}
|
||||
|
||||
// 先删除
|
||||
err = this.Delete(key)
|
||||
err = this.deleteWithoutLocker(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewMemoryWriter(this.valuesMap, key, expiredAt, status, this.locker), nil
|
||||
isWriting = true
|
||||
return NewMemoryWriter(this.valuesMap, key, expiredAt, status, this.locker, func() {
|
||||
this.locker.Lock()
|
||||
delete(this.writingKeyMap, key)
|
||||
this.locker.Unlock()
|
||||
}), nil
|
||||
}
|
||||
|
||||
// Delete 删除某个键值对应的缓存
|
||||
@@ -147,15 +186,12 @@ func (this *MemoryStorage) CleanAll() error {
|
||||
func (this *MemoryStorage) Purge(keys []string, urlType string) error {
|
||||
// 目录
|
||||
if urlType == "dir" {
|
||||
resultKeys := []string{}
|
||||
for _, key := range keys {
|
||||
subKeys, err := this.list.FindKeysWithPrefix(key)
|
||||
err := this.list.CleanPrefix(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resultKeys = append(resultKeys, subKeys...)
|
||||
}
|
||||
keys = resultKeys
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
@@ -170,13 +206,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
|
||||
// Stop 停止缓存策略
|
||||
func (this *MemoryStorage) Stop() {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
this.valuesMap = map[uint64]*MemoryItem{}
|
||||
this.writingKeyMap = map[string]bool{}
|
||||
_ = this.list.Reset()
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
|
||||
_ = this.list.Close()
|
||||
|
||||
this.locker.Unlock()
|
||||
|
||||
remotelogs.Println("CACHE", "close memory storage '"+strconv.FormatInt(this.policy.Id, 10)+"'")
|
||||
}
|
||||
|
||||
// Policy 获取当前存储的Policy
|
||||
@@ -235,3 +277,10 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 {
|
||||
}
|
||||
return c1
|
||||
}
|
||||
|
||||
func (this *MemoryStorage) deleteWithoutLocker(key string) error {
|
||||
hash := this.hash(key)
|
||||
delete(this.valuesMap, hash)
|
||||
_ = this.list.Remove(fmt.Sprintf("%d", hash))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package caches
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -84,6 +87,19 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryStorage_OpenReaderLock(t *testing.T) {
|
||||
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
|
||||
_ = storage.Init()
|
||||
|
||||
var h = storage.hash("test")
|
||||
storage.valuesMap = map[uint64]*MemoryItem{
|
||||
h: {
|
||||
IsDone: true,
|
||||
},
|
||||
}
|
||||
_, _ = storage.OpenReader("test")
|
||||
}
|
||||
|
||||
func TestMemoryStorage_Delete(t *testing.T) {
|
||||
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
|
||||
{
|
||||
@@ -237,3 +253,53 @@ func TestMemoryStorage_Expire(t *testing.T) {
|
||||
}
|
||||
time.Sleep(70 * time.Second)
|
||||
}
|
||||
|
||||
func TestMemoryStorage_Locker(t *testing.T) {
|
||||
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
|
||||
err := storage.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storage.locker.Lock()
|
||||
err = storage.deleteWithoutLocker("a")
|
||||
storage.locker.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestMemoryStorage_Stop(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = map[uint64]*MemoryItem{}
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m[uint64(i)] = &MemoryItem{
|
||||
HeaderValue: []byte("Hello, World"),
|
||||
BodyValue: bytes.Repeat([]byte("Hello"), 1024),
|
||||
}
|
||||
}
|
||||
|
||||
m = map[uint64]*MemoryItem{}
|
||||
|
||||
var before = time.Now()
|
||||
//runtime.GC()
|
||||
debug.FreeOSMemory()
|
||||
/**go func() {
|
||||
time.Sleep(10 * time.Second)
|
||||
runtime.GC()
|
||||
}()**/
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
if stat2.HeapInuse > stat1.HeapInuse {
|
||||
t.Log(stat2.HeapInuse, stat1.HeapInuse, (stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
|
||||
} else {
|
||||
t.Log("0 MB")
|
||||
}
|
||||
|
||||
t.Log(len(m))
|
||||
}
|
||||
|
||||
@@ -1,31 +1,31 @@
|
||||
package caches
|
||||
|
||||
// 缓存内容写入接口
|
||||
// Writer 缓存内容写入接口
|
||||
type Writer interface {
|
||||
// 写入Header数据
|
||||
// WriteHeader 写入Header数据
|
||||
WriteHeader(data []byte) (n int, err error)
|
||||
|
||||
// 写入Body数据
|
||||
// Write 写入Body数据
|
||||
Write(data []byte) (n int, err error)
|
||||
|
||||
// 写入的Header数据大小
|
||||
// HeaderSize 写入的Header数据大小
|
||||
HeaderSize() int64
|
||||
|
||||
// 写入的Body数据大小
|
||||
// BodySize 写入的Body数据大小
|
||||
BodySize() int64
|
||||
|
||||
// 关闭
|
||||
// Close 关闭
|
||||
Close() error
|
||||
|
||||
// 丢弃
|
||||
// Discard 丢弃
|
||||
Discard() error
|
||||
|
||||
// Key
|
||||
// Key Key
|
||||
Key() string
|
||||
|
||||
// 过期时间
|
||||
// ExpiredAt 过期时间
|
||||
ExpiredAt() int64
|
||||
|
||||
// 内容类型
|
||||
// ItemType 内容类型
|
||||
ItemType() ItemType
|
||||
}
|
||||
|
||||
76
internal/caches/writer_compression.go
Normal file
76
internal/caches/writer_compression.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
||||
)
|
||||
|
||||
type compressionWriter struct {
|
||||
rawWriter Writer
|
||||
writer compressions.Writer
|
||||
key string
|
||||
expiredAt int64
|
||||
}
|
||||
|
||||
func NewCompressionWriter(gw Writer, cpWriter compressions.Writer, key string, expiredAt int64) Writer {
|
||||
return &compressionWriter{
|
||||
rawWriter: gw,
|
||||
writer: cpWriter,
|
||||
key: key,
|
||||
expiredAt: expiredAt,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *compressionWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
return this.writer.Write(data)
|
||||
}
|
||||
|
||||
// WriteHeaderLength 写入Header长度数据
|
||||
func (this *compressionWriter) WriteHeaderLength(headerLength int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteBodyLength 写入Body长度数据
|
||||
func (this *compressionWriter) WriteBodyLength(bodyLength int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *compressionWriter) Write(data []byte) (n int, err error) {
|
||||
return this.writer.Write(data)
|
||||
}
|
||||
|
||||
func (this *compressionWriter) Close() error {
|
||||
err := this.writer.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return this.rawWriter.Close()
|
||||
}
|
||||
|
||||
func (this *compressionWriter) Discard() error {
|
||||
err := this.writer.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return this.rawWriter.Discard()
|
||||
}
|
||||
|
||||
func (this *compressionWriter) Key() string {
|
||||
return this.key
|
||||
}
|
||||
|
||||
func (this *compressionWriter) ExpiredAt() int64 {
|
||||
return this.expiredAt
|
||||
}
|
||||
|
||||
func (this *compressionWriter) HeaderSize() int64 {
|
||||
return this.rawWriter.HeaderSize()
|
||||
}
|
||||
|
||||
func (this *compressionWriter) BodySize() int64 {
|
||||
return this.rawWriter.BodySize()
|
||||
}
|
||||
|
||||
// ItemType 内容类型
|
||||
func (this *compressionWriter) ItemType() ItemType {
|
||||
return this.rawWriter.ItemType()
|
||||
}
|
||||
@@ -14,17 +14,19 @@ type FileWriter struct {
|
||||
headerSize int64
|
||||
bodySize int64
|
||||
expiredAt int64
|
||||
endFunc func()
|
||||
}
|
||||
|
||||
func NewFileWriter(rawWriter *os.File, key string, expiredAt int64) *FileWriter {
|
||||
func NewFileWriter(rawWriter *os.File, key string, expiredAt int64, endFunc func()) *FileWriter {
|
||||
return &FileWriter{
|
||||
key: key,
|
||||
rawWriter: rawWriter,
|
||||
expiredAt: expiredAt,
|
||||
endFunc: endFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// 写入数据
|
||||
// WriteHeader 写入数据
|
||||
func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
n, err = this.rawWriter.Write(data)
|
||||
this.headerSize += int64(n)
|
||||
@@ -34,7 +36,7 @@ func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// 写入Header长度数据
|
||||
// WriteHeaderLength 写入Header长度数据
|
||||
func (this *FileWriter) WriteHeaderLength(headerLength int) error {
|
||||
bytes4 := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
|
||||
@@ -51,7 +53,7 @@ func (this *FileWriter) WriteHeaderLength(headerLength int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 写入数据
|
||||
// Write 写入数据
|
||||
func (this *FileWriter) Write(data []byte) (n int, err error) {
|
||||
n, err = this.rawWriter.Write(data)
|
||||
this.bodySize += int64(n)
|
||||
@@ -61,7 +63,7 @@ func (this *FileWriter) Write(data []byte) (n int, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// 写入Body长度数据
|
||||
// WriteBodyLength 写入Body长度数据
|
||||
func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
|
||||
bytes8 := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
|
||||
@@ -78,8 +80,10 @@ func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 关闭
|
||||
// Close 关闭
|
||||
func (this *FileWriter) Close() error {
|
||||
defer this.endFunc()
|
||||
|
||||
err := this.WriteHeaderLength(types.Int(this.headerSize))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -103,8 +107,10 @@ func (this *FileWriter) Close() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// 丢弃
|
||||
// Discard 丢弃
|
||||
func (this *FileWriter) Discard() error {
|
||||
defer this.endFunc()
|
||||
|
||||
_ = this.rawWriter.Close()
|
||||
|
||||
err := os.Remove(this.rawWriter.Name())
|
||||
@@ -127,7 +133,7 @@ func (this *FileWriter) Key() string {
|
||||
return this.key
|
||||
}
|
||||
|
||||
// 内容类型
|
||||
// ItemType 获取内容类型
|
||||
func (this *FileWriter) ItemType() ItemType {
|
||||
return ItemTypeFile
|
||||
}
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package caches
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
)
|
||||
|
||||
type gzipWriter struct {
|
||||
rawWriter Writer
|
||||
writer *gzip.Writer
|
||||
key string
|
||||
expiredAt int64
|
||||
}
|
||||
|
||||
func NewGzipWriter(gw Writer, key string, expiredAt int64) Writer {
|
||||
return &gzipWriter{
|
||||
rawWriter: gw,
|
||||
writer: gzip.NewWriter(gw),
|
||||
key: key,
|
||||
expiredAt: expiredAt,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *gzipWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
return this.writer.Write(data)
|
||||
}
|
||||
|
||||
// 写入Header长度数据
|
||||
func (this *gzipWriter) WriteHeaderLength(headerLength int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 写入Body长度数据
|
||||
func (this *gzipWriter) WriteBodyLength(bodyLength int64) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *gzipWriter) Write(data []byte) (n int, err error) {
|
||||
return this.writer.Write(data)
|
||||
}
|
||||
|
||||
func (this *gzipWriter) Close() error {
|
||||
err := this.writer.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return this.rawWriter.Close()
|
||||
}
|
||||
|
||||
func (this *gzipWriter) Discard() error {
|
||||
err := this.writer.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return this.rawWriter.Discard()
|
||||
}
|
||||
|
||||
func (this *gzipWriter) Key() string {
|
||||
return this.key
|
||||
}
|
||||
|
||||
func (this *gzipWriter) ExpiredAt() int64 {
|
||||
return this.expiredAt
|
||||
}
|
||||
|
||||
func (this *gzipWriter) HeaderSize() int64 {
|
||||
return this.rawWriter.HeaderSize()
|
||||
}
|
||||
|
||||
func (this *gzipWriter) BodySize() int64 {
|
||||
return this.rawWriter.BodySize()
|
||||
}
|
||||
|
||||
// 内容类型
|
||||
func (this *gzipWriter) ItemType() ItemType {
|
||||
return this.rawWriter.ItemType()
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package caches
|
||||
import (
|
||||
"github.com/cespare/xxhash"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MemoryWriter struct {
|
||||
@@ -14,52 +15,59 @@ type MemoryWriter struct {
|
||||
bodySize int64
|
||||
status int
|
||||
|
||||
hash uint64
|
||||
item *MemoryItem
|
||||
hash uint64
|
||||
item *MemoryItem
|
||||
endFunc func()
|
||||
}
|
||||
|
||||
func NewMemoryWriter(m map[uint64]*MemoryItem, key string, expiredAt int64, status int, locker *sync.RWMutex) *MemoryWriter {
|
||||
func NewMemoryWriter(m map[uint64]*MemoryItem, key string, expiredAt int64, status int, locker *sync.RWMutex, endFunc func()) *MemoryWriter {
|
||||
w := &MemoryWriter{
|
||||
m: m,
|
||||
key: key,
|
||||
expiredAt: expiredAt,
|
||||
locker: locker,
|
||||
item: &MemoryItem{
|
||||
ExpiredAt: expiredAt,
|
||||
Status: status,
|
||||
ExpiredAt: expiredAt,
|
||||
ModifiedAt: time.Now().Unix(),
|
||||
Status: status,
|
||||
},
|
||||
status: status,
|
||||
status: status,
|
||||
endFunc: endFunc,
|
||||
}
|
||||
w.hash = w.calculateHash(key)
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
// 写入数据
|
||||
// WriteHeader 写入数据
|
||||
func (this *MemoryWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
this.headerSize += int64(len(data))
|
||||
this.item.HeaderValue = append(this.item.HeaderValue, data...)
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
// 写入数据
|
||||
// Write 写入数据
|
||||
func (this *MemoryWriter) Write(data []byte) (n int, err error) {
|
||||
this.bodySize += int64(len(data))
|
||||
this.item.BodyValue = append(this.item.BodyValue, data...)
|
||||
return len(data), nil
|
||||
}
|
||||
|
||||
// 数据尺寸
|
||||
// HeaderSize 数据尺寸
|
||||
func (this *MemoryWriter) HeaderSize() int64 {
|
||||
return this.headerSize
|
||||
}
|
||||
|
||||
// BodySize 主体内容尺寸
|
||||
func (this *MemoryWriter) BodySize() int64 {
|
||||
return this.bodySize
|
||||
}
|
||||
|
||||
// 关闭
|
||||
// Close 关闭
|
||||
func (this *MemoryWriter) Close() error {
|
||||
// 需要在Locker之外
|
||||
defer this.endFunc()
|
||||
|
||||
if this.item == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -72,25 +80,28 @@ func (this *MemoryWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 丢弃
|
||||
// Discard 丢弃
|
||||
func (this *MemoryWriter) Discard() error {
|
||||
// 需要在Locker之外
|
||||
defer this.endFunc()
|
||||
|
||||
this.locker.Lock()
|
||||
delete(this.m, this.hash)
|
||||
this.locker.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Key
|
||||
// Key 获取Key
|
||||
func (this *MemoryWriter) Key() string {
|
||||
return this.key
|
||||
}
|
||||
|
||||
// 过期时间
|
||||
// ExpiredAt 过期时间
|
||||
func (this *MemoryWriter) ExpiredAt() int64 {
|
||||
return this.expiredAt
|
||||
}
|
||||
|
||||
// 内容类型
|
||||
// ItemType 内容类型
|
||||
func (this *MemoryWriter) ItemType() ItemType {
|
||||
return ItemTypeMemory
|
||||
}
|
||||
|
||||
8
internal/compressions/reader.go
Normal file
8
internal/compressions/reader.go
Normal file
@@ -0,0 +1,8 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
type Reader interface {
|
||||
Read(p []byte) (n int, err error)
|
||||
Close() error
|
||||
}
|
||||
24
internal/compressions/reader_brotli.go
Normal file
24
internal/compressions/reader_brotli.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"github.com/andybalholm/brotli"
|
||||
"io"
|
||||
)
|
||||
|
||||
type BrotliReader struct {
|
||||
reader *brotli.Reader
|
||||
}
|
||||
|
||||
func NewBrotliReader(reader io.Reader) (Reader, error) {
|
||||
return &BrotliReader{reader: brotli.NewReader(reader)}, nil
|
||||
}
|
||||
|
||||
func (this *BrotliReader) Read(p []byte) (n int, err error) {
|
||||
return this.reader.Read(p)
|
||||
}
|
||||
|
||||
func (this *BrotliReader) Close() error {
|
||||
return nil
|
||||
}
|
||||
24
internal/compressions/reader_deflate.go
Normal file
24
internal/compressions/reader_deflate.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"io"
|
||||
)
|
||||
|
||||
type DeflateReader struct {
|
||||
reader io.ReadCloser
|
||||
}
|
||||
|
||||
func NewDeflateReader(reader io.Reader) (Reader, error) {
|
||||
return &DeflateReader{reader: flate.NewReader(reader)}, nil
|
||||
}
|
||||
|
||||
func (this *DeflateReader) Read(p []byte) (n int, err error) {
|
||||
return this.reader.Read(p)
|
||||
}
|
||||
|
||||
func (this *DeflateReader) Close() error {
|
||||
return this.reader.Close()
|
||||
}
|
||||
30
internal/compressions/reader_gzip.go
Normal file
30
internal/compressions/reader_gzip.go
Normal file
@@ -0,0 +1,30 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
)
|
||||
|
||||
type GzipReader struct {
|
||||
reader *gzip.Reader
|
||||
}
|
||||
|
||||
func NewGzipReader(reader io.Reader) (Reader, error) {
|
||||
r, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &GzipReader{
|
||||
reader: r,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *GzipReader) Read(p []byte) (n int, err error) {
|
||||
return this.reader.Read(p)
|
||||
}
|
||||
|
||||
func (this *GzipReader) Close() error {
|
||||
return this.reader.Close()
|
||||
}
|
||||
46
internal/compressions/utils.go
Normal file
46
internal/compressions/utils.go
Normal file
@@ -0,0 +1,46 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ContentEncoding = string
|
||||
|
||||
const (
|
||||
ContentEncodingBr ContentEncoding = "br"
|
||||
ContentEncodingGzip ContentEncoding = "gzip"
|
||||
ContentEncodingDeflate ContentEncoding = "deflate"
|
||||
)
|
||||
|
||||
var ErrNotSupportedContentEncoding = errors.New("not supported content encoding")
|
||||
|
||||
// NewReader 获取Reader
|
||||
func NewReader(reader io.Reader, contentEncoding ContentEncoding) (Reader, error) {
|
||||
switch contentEncoding {
|
||||
case ContentEncodingBr:
|
||||
return NewBrotliReader(reader)
|
||||
case ContentEncodingGzip:
|
||||
return NewGzipReader(reader)
|
||||
case ContentEncodingDeflate:
|
||||
return NewDeflateReader(reader)
|
||||
}
|
||||
return nil, ErrNotSupportedContentEncoding
|
||||
}
|
||||
|
||||
// NewWriter 获取Writer
|
||||
// TODO 考虑重用Writer
|
||||
func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType, level int) (Writer, error) {
|
||||
switch compressType {
|
||||
case serverconfigs.HTTPCompressionTypeGzip:
|
||||
return NewGzipWriter(writer, level)
|
||||
case serverconfigs.HTTPCompressionTypeDeflate:
|
||||
return NewDeflateWriter(writer, level)
|
||||
case serverconfigs.HTTPCompressionTypeBrotli:
|
||||
return NewBrotliWriter(writer, level)
|
||||
}
|
||||
return nil, errors.New("invalid compression type '" + compressType + "'")
|
||||
}
|
||||
10
internal/compressions/writer.go
Normal file
10
internal/compressions/writer.go
Normal file
@@ -0,0 +1,10 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
type Writer interface {
|
||||
Write(p []byte) (int, error)
|
||||
Flush() error
|
||||
Close() error
|
||||
Level() int
|
||||
}
|
||||
41
internal/compressions/writer_brotli.go
Normal file
41
internal/compressions/writer_brotli.go
Normal file
@@ -0,0 +1,41 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"github.com/andybalholm/brotli"
|
||||
"io"
|
||||
)
|
||||
|
||||
type BrotliWriter struct {
|
||||
writer *brotli.Writer
|
||||
level int
|
||||
}
|
||||
|
||||
func NewBrotliWriter(writer io.Writer, level int) (Writer, error) {
|
||||
if level <= 0 {
|
||||
level = brotli.BestSpeed
|
||||
} else if level > brotli.BestCompression {
|
||||
level = brotli.BestCompression
|
||||
}
|
||||
return &BrotliWriter{
|
||||
writer: brotli.NewWriterLevel(writer, level),
|
||||
level: level,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *BrotliWriter) Write(p []byte) (int, error) {
|
||||
return this.writer.Write(p)
|
||||
}
|
||||
|
||||
func (this *BrotliWriter) Flush() error {
|
||||
return this.writer.Flush()
|
||||
}
|
||||
|
||||
func (this *BrotliWriter) Close() error {
|
||||
return this.writer.Close()
|
||||
}
|
||||
|
||||
func (this *BrotliWriter) Level() int {
|
||||
return this.level
|
||||
}
|
||||
47
internal/compressions/writer_deflate.go
Normal file
47
internal/compressions/writer_deflate.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"io"
|
||||
)
|
||||
|
||||
type DeflateWriter struct {
|
||||
writer *flate.Writer
|
||||
level int
|
||||
}
|
||||
|
||||
func NewDeflateWriter(writer io.Writer, level int) (Writer, error) {
|
||||
if level <= 0 {
|
||||
level = flate.BestSpeed
|
||||
} else if level > flate.BestCompression {
|
||||
level = flate.BestCompression
|
||||
}
|
||||
|
||||
flateWriter, err := flate.NewWriter(writer, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &DeflateWriter{
|
||||
writer: flateWriter,
|
||||
level: level,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *DeflateWriter) Write(p []byte) (int, error) {
|
||||
return this.writer.Write(p)
|
||||
}
|
||||
|
||||
func (this *DeflateWriter) Flush() error {
|
||||
return this.writer.Flush()
|
||||
}
|
||||
|
||||
func (this *DeflateWriter) Close() error {
|
||||
return this.writer.Close()
|
||||
}
|
||||
|
||||
func (this *DeflateWriter) Level() int {
|
||||
return this.level
|
||||
}
|
||||
51
internal/compressions/writer_encoding.go
Normal file
51
internal/compressions/writer_encoding.go
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
)
|
||||
|
||||
type EncodingWriter struct {
|
||||
contentEncoding ContentEncoding
|
||||
writer Writer
|
||||
buf *bytes.Buffer
|
||||
}
|
||||
|
||||
func NewEncodingWriter(contentEncoding ContentEncoding, writer Writer) (Writer, error) {
|
||||
return &EncodingWriter{
|
||||
contentEncoding: contentEncoding,
|
||||
writer: writer,
|
||||
buf: &bytes.Buffer{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *EncodingWriter) Write(p []byte) (int, error) {
|
||||
return this.buf.Write(p)
|
||||
}
|
||||
|
||||
func (this *EncodingWriter) Flush() error {
|
||||
return this.writer.Flush()
|
||||
}
|
||||
|
||||
func (this *EncodingWriter) Close() error {
|
||||
reader, err := NewReader(this.buf, this.contentEncoding)
|
||||
if err != nil {
|
||||
_ = this.writer.Close()
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(this.writer, reader)
|
||||
if err != nil {
|
||||
_ = reader.Close()
|
||||
_ = this.writer.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
_ = reader.Close()
|
||||
return this.writer.Close()
|
||||
}
|
||||
|
||||
func (this *EncodingWriter) Level() int {
|
||||
return this.writer.Level()
|
||||
}
|
||||
47
internal/compressions/writer_encoding_test.go
Normal file
47
internal/compressions/writer_encoding_test.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewEncodingWriter(t *testing.T) {
|
||||
var buf = &bytes.Buffer{}
|
||||
|
||||
subWriter, err := NewWriter(buf, serverconfigs.HTTPCompressionTypeGzip, 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer, err := NewEncodingWriter(ContentEncodingGzip, subWriter)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gzipBuf := &bytes.Buffer{}
|
||||
gzipWriter, err := NewGzipWriter(gzipBuf, 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = gzipWriter.Write([]byte("Hello"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = gzipWriter.Write([]byte("World"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = gzipWriter.Close()
|
||||
|
||||
_, err = writer.Write(gzipBuf.Bytes())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_ = writer.Close()
|
||||
|
||||
t.Log(buf.String())
|
||||
}
|
||||
47
internal/compressions/writer_gzip.go
Normal file
47
internal/compressions/writer_gzip.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
)
|
||||
|
||||
type GzipWriter struct {
|
||||
writer *gzip.Writer
|
||||
level int
|
||||
}
|
||||
|
||||
func NewGzipWriter(writer io.Writer, level int) (Writer, error) {
|
||||
if level <= 0 {
|
||||
level = gzip.BestSpeed
|
||||
} else if level > gzip.BestCompression {
|
||||
level = gzip.BestCompression
|
||||
}
|
||||
|
||||
gzipWriter, err := gzip.NewWriterLevel(writer, level)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &GzipWriter{
|
||||
writer: gzipWriter,
|
||||
level: level,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *GzipWriter) Write(p []byte) (int, error) {
|
||||
return this.writer.Write(p)
|
||||
}
|
||||
|
||||
func (this *GzipWriter) Flush() error {
|
||||
return this.writer.Flush()
|
||||
}
|
||||
|
||||
func (this *GzipWriter) Close() error {
|
||||
return this.writer.Close()
|
||||
}
|
||||
|
||||
func (this *GzipWriter) Level() int {
|
||||
return this.level
|
||||
}
|
||||
8
internal/const/build.go
Normal file
8
internal/const/build.go
Normal file
@@ -0,0 +1,8 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// +build community
|
||||
|
||||
package teaconst
|
||||
|
||||
const BuildCommunity = true
|
||||
const BuildPlus = false
|
||||
const Tag = "community"
|
||||
8
internal/const/build_plus.go
Normal file
8
internal/const/build_plus.go
Normal file
@@ -0,0 +1,8 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// +build plus
|
||||
|
||||
package teaconst
|
||||
|
||||
const BuildCommunity = false
|
||||
const BuildPlus = true
|
||||
const Tag = "plus"
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "0.1.1"
|
||||
Version = "0.3.3"
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
@@ -10,7 +10,7 @@ const (
|
||||
IPItemTypeAll IPItemType = "all" // 所有IP
|
||||
)
|
||||
|
||||
// IP条目
|
||||
// IPItem IP条目
|
||||
type IPItem struct {
|
||||
Type string `json:"type"`
|
||||
Id int64 `json:"id"`
|
||||
@@ -20,7 +20,7 @@ type IPItem struct {
|
||||
EventLevel string `json:"eventLevel"`
|
||||
}
|
||||
|
||||
// 检查是否包含某个IP
|
||||
// Contains 检查是否包含某个IP
|
||||
func (this *IPItem) Contains(ip uint64) bool {
|
||||
switch this.Type {
|
||||
case IPItemTypeIPv4:
|
||||
|
||||
@@ -3,6 +3,7 @@ package iplibrary
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -72,3 +73,36 @@ func TestIPItem_Contains(t *testing.T) {
|
||||
a.IsTrue(item.Contains(utils.IP2Long("192.168.1.1")))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPItem_Memory(t *testing.T) {
|
||||
var list = NewIPList()
|
||||
for i := 0; i < 2_000_000; i ++ {
|
||||
list.Add(&IPItem{
|
||||
Type: "ip",
|
||||
Id: int64(i),
|
||||
IPFrom: utils.IP2Long("192.168.1.1"),
|
||||
IPTo: 0,
|
||||
ExpiredAt: time.Now().Unix(),
|
||||
EventLevel: "",
|
||||
})
|
||||
}
|
||||
t.Log("waiting")
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
func BenchmarkIPItem_Contains(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
item := &IPItem{
|
||||
IPFrom: utils.IP2Long("192.168.1.1"),
|
||||
IPTo: utils.IP2Long("192.168.1.101"),
|
||||
ExpiredAt: 0,
|
||||
}
|
||||
ip := utils.IP2Long("192.168.1.1")
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := 0; j < 10_000; j++ {
|
||||
item.Contains(ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,24 +3,26 @@ package iplibrary
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/expires"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// IP名单
|
||||
// IPList IP名单
|
||||
// TODO IP名单可以分片关闭,这样让每一片的数据量减少,查询更快
|
||||
type IPList struct {
|
||||
itemsMap map[int64]*IPItem // id => item
|
||||
ipMap map[uint64][]int64 // ip => itemIds
|
||||
expireList *expires.List
|
||||
itemsMap map[int64]*IPItem // id => item
|
||||
sortedItems []*IPItem
|
||||
allItemsMap map[int64]*IPItem // id => item
|
||||
|
||||
isAll bool
|
||||
expireList *expires.List
|
||||
|
||||
locker sync.RWMutex
|
||||
}
|
||||
|
||||
func NewIPList() *IPList {
|
||||
list := &IPList{
|
||||
itemsMap: map[int64]*IPItem{},
|
||||
ipMap: map[uint64][]int64{},
|
||||
itemsMap: map[int64]*IPItem{},
|
||||
allItemsMap: map[int64]*IPItem{},
|
||||
}
|
||||
|
||||
expireList := expires.NewList()
|
||||
@@ -34,14 +36,94 @@ func NewIPList() *IPList {
|
||||
}
|
||||
|
||||
func (this *IPList) Add(item *IPItem) {
|
||||
this.addItem(item, true)
|
||||
}
|
||||
|
||||
// AddDelay 延迟添加,需要手工调用Sort()函数
|
||||
func (this *IPList) AddDelay(item *IPItem) {
|
||||
this.addItem(item, false)
|
||||
}
|
||||
|
||||
func (this *IPList) Sort() {
|
||||
this.locker.Lock()
|
||||
this.sortItems()
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *IPList) Delete(itemId int64) {
|
||||
this.locker.Lock()
|
||||
this.deleteItem(itemId)
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
// Contains 判断是否包含某个IP
|
||||
func (this *IPList) Contains(ip uint64) bool {
|
||||
this.locker.RLock()
|
||||
if len(this.allItemsMap) > 0 {
|
||||
this.locker.RUnlock()
|
||||
return true
|
||||
}
|
||||
|
||||
var item = this.lookupIP(ip)
|
||||
|
||||
this.locker.RUnlock()
|
||||
|
||||
return item != nil
|
||||
}
|
||||
|
||||
// ContainsIPStrings 是否包含一组IP中的任意一个,并返回匹配的第一个Item
|
||||
func (this *IPList) ContainsIPStrings(ipStrings []string) (item *IPItem, found bool) {
|
||||
if len(ipStrings) == 0 {
|
||||
return
|
||||
}
|
||||
this.locker.RLock()
|
||||
if len(this.allItemsMap) > 0 {
|
||||
for _, allItem := range this.allItemsMap {
|
||||
item = allItem
|
||||
break
|
||||
}
|
||||
|
||||
if item != nil {
|
||||
this.locker.RUnlock()
|
||||
found = true
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, ipString := range ipStrings {
|
||||
if len(ipString) == 0 {
|
||||
continue
|
||||
}
|
||||
item = this.lookupIP(utils.IP2Long(ipString))
|
||||
if item != nil {
|
||||
this.locker.RUnlock()
|
||||
found = true
|
||||
return
|
||||
}
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *IPList) addItem(item *IPItem, sortable bool) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if item.ExpiredAt > 0 && item.ExpiredAt < utils.UnixTime() {
|
||||
return
|
||||
}
|
||||
|
||||
if item.IPFrom == 0 && item.IPTo == 0 {
|
||||
if item.Type != "all" {
|
||||
if item.Type != IPItemTypeAll {
|
||||
return
|
||||
}
|
||||
} else if item.IPTo > 0 {
|
||||
if item.IPFrom > item.IPTo {
|
||||
item.IPFrom, item.IPTo = item.IPTo, item.IPFrom
|
||||
} else if item.IPFrom == 0 {
|
||||
item.IPFrom = item.IPTo
|
||||
item.IPTo = 0
|
||||
}
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
@@ -56,157 +138,86 @@ func (this *IPList) Add(item *IPItem) {
|
||||
|
||||
// 展开
|
||||
if item.IPFrom > 0 {
|
||||
if item.IPTo == 0 {
|
||||
this.addIP(item.IPFrom, item.Id)
|
||||
} else {
|
||||
if item.IPFrom > item.IPTo {
|
||||
item.IPTo, item.IPFrom = item.IPFrom, item.IPTo
|
||||
}
|
||||
|
||||
for i := item.IPFrom; i <= item.IPTo; i++ {
|
||||
// 最多不能超过65535,防止整个系统内存爆掉
|
||||
if i >= item.IPFrom+65535 {
|
||||
break
|
||||
}
|
||||
this.addIP(i, item.Id)
|
||||
}
|
||||
}
|
||||
} else if item.IPTo > 0 {
|
||||
this.addIP(item.IPTo, item.Id)
|
||||
this.sortedItems = append(this.sortedItems, item)
|
||||
} else {
|
||||
this.addIP(0, item.Id)
|
||||
|
||||
// 更新isAll
|
||||
this.isAll = true
|
||||
this.allItemsMap[item.Id] = item
|
||||
}
|
||||
|
||||
if item.ExpiredAt > 0 {
|
||||
this.expireList.Add(item.Id, item.ExpiredAt)
|
||||
}
|
||||
|
||||
if sortable {
|
||||
this.sortItems()
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *IPList) Delete(itemId int64) {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
this.deleteItem(itemId)
|
||||
|
||||
// 更新isAll
|
||||
this.isAll = len(this.ipMap[0]) > 0
|
||||
// 对列表进行排序
|
||||
func (this *IPList) sortItems() {
|
||||
sort.Slice(this.sortedItems, func(i, j int) bool {
|
||||
var item1 = this.sortedItems[i]
|
||||
var item2 = this.sortedItems[j]
|
||||
if item1.IPFrom == item2.IPFrom {
|
||||
return item1.IPTo < item2.IPTo
|
||||
}
|
||||
return item1.IPFrom < item2.IPFrom
|
||||
})
|
||||
}
|
||||
|
||||
// 判断是否包含某个IP
|
||||
func (this *IPList) Contains(ip uint64) bool {
|
||||
this.locker.RLock()
|
||||
if this.isAll {
|
||||
this.locker.RUnlock()
|
||||
return true
|
||||
}
|
||||
_, ok := this.ipMap[ip]
|
||||
this.locker.RUnlock()
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
// 是否包含一组IP
|
||||
func (this *IPList) ContainsIPStrings(ipStrings []string) (found bool, item *IPItem) {
|
||||
if len(ipStrings) == 0 {
|
||||
return
|
||||
}
|
||||
this.locker.RLock()
|
||||
if this.isAll {
|
||||
itemIds := this.ipMap[0]
|
||||
if len(itemIds) > 0 {
|
||||
itemId := itemIds[0]
|
||||
item = this.itemsMap[itemId]
|
||||
}
|
||||
|
||||
this.locker.RUnlock()
|
||||
found = true
|
||||
return
|
||||
}
|
||||
for _, ipString := range ipStrings {
|
||||
if len(ipString) == 0 {
|
||||
continue
|
||||
}
|
||||
itemIds, ok := this.ipMap[utils.IP2Long(ipString)]
|
||||
if ok {
|
||||
if len(itemIds) > 0 {
|
||||
itemId := itemIds[0]
|
||||
item = this.itemsMap[itemId]
|
||||
// 不加锁的情况下查找Item
|
||||
func (this *IPList) lookupIP(ip uint64) *IPItem {
|
||||
var count = len(this.sortedItems)
|
||||
var resultIndex = -1
|
||||
sort.Search(count, func(i int) bool {
|
||||
var item = this.sortedItems[i]
|
||||
if item.IPFrom < ip {
|
||||
if item.IPTo >= ip {
|
||||
resultIndex = i
|
||||
}
|
||||
|
||||
this.locker.RUnlock()
|
||||
found = true
|
||||
return
|
||||
return false
|
||||
} else if item.IPFrom == ip {
|
||||
resultIndex = i
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if resultIndex < 0 || resultIndex >= count {
|
||||
return nil
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
|
||||
return this.sortedItems[resultIndex]
|
||||
}
|
||||
|
||||
// 在不加锁的情况下删除某个Item
|
||||
// 将会被别的方法引用,切记不能加锁
|
||||
func (this *IPList) deleteItem(itemId int64) {
|
||||
item, ok := this.itemsMap[itemId]
|
||||
_, ok := this.itemsMap[itemId]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
delete(this.itemsMap, itemId)
|
||||
|
||||
// 展开
|
||||
if item.IPFrom > 0 {
|
||||
if item.IPTo == 0 {
|
||||
this.deleteIP(item.IPFrom, item.Id)
|
||||
} else {
|
||||
if item.IPFrom > item.IPTo {
|
||||
item.IPTo, item.IPFrom = item.IPFrom, item.IPTo
|
||||
}
|
||||
|
||||
for i := item.IPFrom; i <= item.IPTo; i++ {
|
||||
// 最多不能超过65535,防止整个系统内存爆掉
|
||||
if i >= item.IPFrom+65535 {
|
||||
break
|
||||
}
|
||||
this.deleteIP(i, item.Id)
|
||||
}
|
||||
}
|
||||
} else if item.IPTo > 0 {
|
||||
this.deleteIP(item.IPTo, item.Id)
|
||||
} else {
|
||||
this.deleteIP(0, item.Id)
|
||||
}
|
||||
}
|
||||
|
||||
// 添加单个IP
|
||||
func (this *IPList) addIP(ip uint64, itemId int64) {
|
||||
itemIds, ok := this.ipMap[ip]
|
||||
// 是否为All Item
|
||||
_, ok = this.allItemsMap[itemId]
|
||||
if ok {
|
||||
itemIds = append(itemIds, itemId)
|
||||
} else {
|
||||
itemIds = []int64{itemId}
|
||||
}
|
||||
this.ipMap[ip] = itemIds
|
||||
}
|
||||
|
||||
// 删除单个IP
|
||||
func (this *IPList) deleteIP(ip uint64, itemId int64) {
|
||||
itemIds, ok := this.ipMap[ip]
|
||||
if !ok {
|
||||
delete(this.allItemsMap, itemId)
|
||||
return
|
||||
}
|
||||
newItemIds := []int64{}
|
||||
for _, oldItemId := range itemIds {
|
||||
if oldItemId == itemId {
|
||||
continue
|
||||
|
||||
// 删除排序中的Item
|
||||
var index = -1
|
||||
for itemIndex, item := range this.sortedItems {
|
||||
if item.Id == itemId {
|
||||
index = itemIndex
|
||||
break
|
||||
}
|
||||
newItemIds = append(newItemIds, oldItemId)
|
||||
}
|
||||
if len(newItemIds) > 0 {
|
||||
this.ipMap[ip] = newItemIds
|
||||
} else {
|
||||
delete(this.ipMap, ip)
|
||||
if index >= 0 {
|
||||
copy(this.sortedItems[index:], this.sortedItems[index+1:])
|
||||
this.sortedItems = this.sortedItems[:len(this.sortedItems)-1]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
@@ -16,7 +17,7 @@ func TestIPList_Add_Empty(t *testing.T) {
|
||||
Id: 1,
|
||||
})
|
||||
logs.PrintAsJSON(ipList.itemsMap, t)
|
||||
logs.PrintAsJSON(ipList.ipMap, t)
|
||||
logs.PrintAsJSON(ipList.allItemsMap, t)
|
||||
}
|
||||
|
||||
func TestIPList_Add_One(t *testing.T) {
|
||||
@@ -31,15 +32,30 @@ func TestIPList_Add_One(t *testing.T) {
|
||||
})
|
||||
ipList.Add(&IPItem{
|
||||
Id: 3,
|
||||
IPFrom: utils.IP2Long("2001:db8:0:1::101"),
|
||||
IPFrom: utils.IP2Long("192.168.0.2"),
|
||||
})
|
||||
ipList.Add(&IPItem{
|
||||
Id: 4,
|
||||
IPFrom: utils.IP2Long("192.168.0.2"),
|
||||
IPTo: utils.IP2Long("192.168.0.1"),
|
||||
})
|
||||
ipList.Add(&IPItem{
|
||||
Id: 5,
|
||||
IPFrom: utils.IP2Long("2001:db8:0:1::101"),
|
||||
})
|
||||
ipList.Add(&IPItem{
|
||||
Id: 6,
|
||||
IPFrom: 0,
|
||||
Type: "all",
|
||||
})
|
||||
t.Log("===items===")
|
||||
logs.PrintAsJSON(ipList.itemsMap, t)
|
||||
logs.PrintAsJSON(ipList.ipMap, t) // ip => items
|
||||
|
||||
t.Log("===sorted items===")
|
||||
logs.PrintAsJSON(ipList.sortedItems, t)
|
||||
|
||||
t.Log("===all items===")
|
||||
logs.PrintAsJSON(ipList.allItemsMap, t) // ip => items
|
||||
}
|
||||
|
||||
func TestIPList_Update(t *testing.T) {
|
||||
@@ -50,14 +66,31 @@ func TestIPList_Update(t *testing.T) {
|
||||
})
|
||||
/**ipList.Add(&IPItem{
|
||||
Id: 2,
|
||||
IPFrom: IP2Long("192.168.1.1"),
|
||||
IPFrom: utils.IP2Long("192.168.1.1"),
|
||||
})**/
|
||||
ipList.Add(&IPItem{
|
||||
Id: 1,
|
||||
IPTo: utils.IP2Long("192.168.1.2"),
|
||||
})
|
||||
logs.PrintAsJSON(ipList.itemsMap, t)
|
||||
logs.PrintAsJSON(ipList.ipMap, t)
|
||||
logs.PrintAsJSON(ipList.sortedItems, t)
|
||||
}
|
||||
|
||||
func TestIPList_Update_AllItems(t *testing.T) {
|
||||
ipList := NewIPList()
|
||||
ipList.Add(&IPItem{
|
||||
Id: 1,
|
||||
Type: IPItemTypeAll,
|
||||
IPFrom: 0,
|
||||
})
|
||||
ipList.Add(&IPItem{
|
||||
Id: 1,
|
||||
IPTo: 0,
|
||||
})
|
||||
t.Log("===items map===")
|
||||
logs.PrintAsJSON(ipList.itemsMap, t)
|
||||
t.Log("===all items map===")
|
||||
logs.PrintAsJSON(ipList.allItemsMap, t)
|
||||
}
|
||||
|
||||
func TestIPList_Add_Range(t *testing.T) {
|
||||
@@ -71,9 +104,9 @@ func TestIPList_Add_Range(t *testing.T) {
|
||||
Id: 2,
|
||||
IPTo: utils.IP2Long("192.168.1.2"),
|
||||
})
|
||||
t.Log(len(ipList.ipMap), "ips")
|
||||
t.Log(len(ipList.itemsMap), "ips")
|
||||
logs.PrintAsJSON(ipList.itemsMap, t)
|
||||
logs.PrintAsJSON(ipList.ipMap, t)
|
||||
logs.PrintAsJSON(ipList.allItemsMap, t)
|
||||
}
|
||||
|
||||
func TestIPList_Add_Overflow(t *testing.T) {
|
||||
@@ -85,8 +118,8 @@ func TestIPList_Add_Overflow(t *testing.T) {
|
||||
IPFrom: utils.IP2Long("192.168.1.1"),
|
||||
IPTo: utils.IP2Long("192.169.255.1"),
|
||||
})
|
||||
t.Log(len(ipList.ipMap), "ips")
|
||||
a.IsTrue(len(ipList.ipMap) <= 65535)
|
||||
t.Log(len(ipList.itemsMap), "ips")
|
||||
a.IsTrue(len(ipList.itemsMap) <= 65535)
|
||||
}
|
||||
|
||||
func TestNewIPList_Memory(t *testing.T) {
|
||||
@@ -104,20 +137,50 @@ func TestNewIPList_Memory(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIPList_Contains(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
list := NewIPList()
|
||||
for i := 0; i < 255; i++ {
|
||||
list.Add(&IPItem{
|
||||
list.AddDelay(&IPItem{
|
||||
Id: int64(i),
|
||||
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
|
||||
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
|
||||
ExpiredAt: 0,
|
||||
})
|
||||
}
|
||||
t.Log(len(list.ipMap), "ip")
|
||||
for i := 0; i < 255; i++ {
|
||||
list.AddDelay(&IPItem{
|
||||
Id: int64(1000 + i),
|
||||
IPFrom: utils.IP2Long("192.167.2." + strconv.Itoa(i)),
|
||||
})
|
||||
}
|
||||
list.Sort()
|
||||
t.Log(len(list.itemsMap), "ip")
|
||||
|
||||
before := time.Now()
|
||||
t.Log(list.Contains(utils.IP2Long("192.168.1.100")))
|
||||
t.Log(list.Contains(utils.IP2Long("192.168.2.100")))
|
||||
a.IsTrue(list.Contains(utils.IP2Long("192.168.1.100")))
|
||||
a.IsTrue(list.Contains(utils.IP2Long("192.168.2.100")))
|
||||
a.IsFalse(list.Contains(utils.IP2Long("192.169.3.100")))
|
||||
a.IsFalse(list.Contains(utils.IP2Long("192.167.3.100")))
|
||||
a.IsTrue(list.Contains(utils.IP2Long("192.167.2.100")))
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
func TestIPList_Contains_Many(t *testing.T) {
|
||||
list := NewIPList()
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
list.AddDelay(&IPItem{
|
||||
Id: int64(i),
|
||||
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
|
||||
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
|
||||
ExpiredAt: 0,
|
||||
})
|
||||
}
|
||||
list.Sort()
|
||||
t.Log(len(list.itemsMap), "ip")
|
||||
|
||||
before := time.Now()
|
||||
_ = list.Contains(utils.IP2Long("192.168.1.100"))
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
@@ -146,6 +209,32 @@ func TestIPList_ContainsAll(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestIPList_ContainsIPStrings(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
list := NewIPList()
|
||||
for i := 0; i < 255; i++ {
|
||||
list.Add(&IPItem{
|
||||
Id: int64(i),
|
||||
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
|
||||
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
|
||||
ExpiredAt: 0,
|
||||
})
|
||||
}
|
||||
t.Log(len(list.itemsMap), "ip")
|
||||
|
||||
{
|
||||
item, ok := list.ContainsIPStrings([]string{"192.168.1.100"})
|
||||
t.Log("item:", item)
|
||||
a.IsTrue(ok)
|
||||
}
|
||||
{
|
||||
item, ok := list.ContainsIPStrings([]string{"192.167.1.100"})
|
||||
t.Log("item:", item)
|
||||
a.IsFalse(ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPList_Delete(t *testing.T) {
|
||||
list := NewIPList()
|
||||
list.Add(&IPItem{
|
||||
@@ -160,13 +249,13 @@ func TestIPList_Delete(t *testing.T) {
|
||||
})
|
||||
t.Log("===BEFORE===")
|
||||
logs.PrintAsJSON(list.itemsMap, t)
|
||||
logs.PrintAsJSON(list.ipMap, t)
|
||||
logs.PrintAsJSON(list.allItemsMap, t)
|
||||
|
||||
list.Delete(1)
|
||||
|
||||
t.Log("===AFTER===")
|
||||
logs.PrintAsJSON(list.itemsMap, t)
|
||||
logs.PrintAsJSON(list.ipMap, t)
|
||||
logs.PrintAsJSON(list.allItemsMap, t)
|
||||
}
|
||||
|
||||
func TestGC(t *testing.T) {
|
||||
@@ -184,27 +273,27 @@ func TestGC(t *testing.T) {
|
||||
ExpiredAt: 0,
|
||||
})
|
||||
logs.PrintAsJSON(list.itemsMap, t)
|
||||
logs.PrintAsJSON(list.ipMap, t)
|
||||
logs.PrintAsJSON(list.allItemsMap, t)
|
||||
|
||||
time.Sleep(2 * time.Second)
|
||||
t.Log("===AFTER GC===")
|
||||
logs.PrintAsJSON(list.itemsMap, t)
|
||||
logs.PrintAsJSON(list.ipMap, t)
|
||||
logs.PrintAsJSON(list.sortedItems, t)
|
||||
}
|
||||
|
||||
func BenchmarkIPList_Contains(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
list := NewIPList()
|
||||
for i := 192; i < 194; i++ {
|
||||
for i := 1; i < 194; i++ {
|
||||
list.Add(&IPItem{
|
||||
Id: int64(1),
|
||||
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".1.0.1"),
|
||||
IPTo: utils.IP2Long(strconv.Itoa(i) + ".2.0.1"),
|
||||
Id: int64(i),
|
||||
IPFrom: utils.IP2Long(strconv.Itoa(i%255) + "." + strconv.Itoa(i%255) + ".0.1"),
|
||||
IPTo: utils.IP2Long(strconv.Itoa(i%255) + "." + strconv.Itoa(i%255) + ".0.1"),
|
||||
ExpiredAt: time.Now().Unix() + 60,
|
||||
})
|
||||
}
|
||||
b.Log(len(list.ipMap), "ip")
|
||||
b.Log(len(list.itemsMap), "ip")
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = list.Contains(utils.IP2Long("192.168.1.100"))
|
||||
}
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
package iplibrary
|
||||
|
||||
type LibraryInterface interface {
|
||||
// 加载数据库文件
|
||||
// Load 加载数据库文件
|
||||
Load(dbPath string) error
|
||||
|
||||
// 查询IP
|
||||
// Lookup 查询IP
|
||||
// 返回结果有可能为空
|
||||
Lookup(ip string) (*Result, error)
|
||||
|
||||
// 关闭数据库文件
|
||||
// Close 关闭数据库文件
|
||||
Close()
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/lionsoul2014/ip2region/binding/golang/ip2region"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -27,6 +28,9 @@ func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
|
||||
if strings.Contains(ip, ":") {
|
||||
return nil, nil
|
||||
}
|
||||
if net.ParseIP(ip) == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if this.db == nil {
|
||||
return nil, errors.New("library has not been loaded")
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/files"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -21,7 +21,7 @@ func init() {
|
||||
// 初始化
|
||||
library, err := SharedManager.Load()
|
||||
if err != nil {
|
||||
logs.Println("[IP_LIBRARY]" + err.Error())
|
||||
remotelogs.Error("IP_LIBRARY", err.Error())
|
||||
return
|
||||
}
|
||||
SharedLibrary = library
|
||||
|
||||
@@ -20,12 +20,12 @@ import (
|
||||
var SharedCountryManager = NewCountryManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
go SharedCountryManager.Start()
|
||||
})
|
||||
}
|
||||
|
||||
// 国家信息管理
|
||||
// CountryManager 国家/地区信息管理
|
||||
type CountryManager struct {
|
||||
cacheFile string
|
||||
|
||||
|
||||
@@ -15,12 +15,12 @@ var SharedIPListManager = NewIPListManager()
|
||||
var IPListUpdateNotify = make(chan bool, 1)
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
go SharedIPListManager.Start()
|
||||
})
|
||||
}
|
||||
|
||||
// IP名单管理
|
||||
// IPListManager IP名单管理
|
||||
type IPListManager struct {
|
||||
// 缓存文件
|
||||
// 每行一个数据:id|from|to|expiredAt
|
||||
@@ -112,12 +112,16 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
return false, nil
|
||||
}
|
||||
this.locker.Lock()
|
||||
var changedLists = map[*IPList]bool{}
|
||||
for _, item := range items {
|
||||
list, ok := this.listMap[item.ListId]
|
||||
if !ok {
|
||||
list = NewIPList()
|
||||
this.listMap[item.ListId] = list
|
||||
}
|
||||
|
||||
changedLists[list] = true
|
||||
|
||||
if item.IsDeleted {
|
||||
list.Delete(item.Id)
|
||||
|
||||
@@ -127,7 +131,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
continue
|
||||
}
|
||||
|
||||
list.Add(&IPItem{
|
||||
list.AddDelay(&IPItem{
|
||||
Id: item.Id,
|
||||
Type: item.Type,
|
||||
IPFrom: utils.IP2Long(item.IpFrom),
|
||||
@@ -140,6 +144,11 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
SharedActionManager.DeleteItem(item.ListType, item)
|
||||
SharedActionManager.AddItem(item.ListType, item)
|
||||
}
|
||||
|
||||
for changedList := range changedLists {
|
||||
changedList.Sort()
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
this.version = items[len(items)-1].Version
|
||||
|
||||
|
||||
@@ -24,12 +24,12 @@ const (
|
||||
var SharedProvinceManager = NewProvinceManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
go SharedProvinceManager.Start()
|
||||
})
|
||||
}
|
||||
|
||||
// 国家信息管理
|
||||
// ProvinceManager 中国省份信息管理
|
||||
type ProvinceManager struct {
|
||||
cacheFile string
|
||||
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
@@ -21,16 +21,16 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
// IP库更新程序
|
||||
// Updater IP库更新程序
|
||||
type Updater struct {
|
||||
}
|
||||
|
||||
// 获取新对象
|
||||
// NewUpdater 获取新对象
|
||||
func NewUpdater() *Updater {
|
||||
return &Updater{}
|
||||
}
|
||||
|
||||
// 开始更新
|
||||
// Start 开始更新
|
||||
func (this *Updater) Start() {
|
||||
// 这里不需要太频繁检查更新,因为通常不需要更新IP库
|
||||
ticker := time.NewTicker(1 * time.Hour)
|
||||
@@ -38,7 +38,7 @@ func (this *Updater) Start() {
|
||||
for range ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
logs.Println("[IP_LIBRARY]" + err.Error())
|
||||
remotelogs.Error("IP_LIBRARY", err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
50
internal/js/console.go
Normal file
50
internal/js/console.go
Normal file
@@ -0,0 +1,50 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type Console struct {
|
||||
}
|
||||
|
||||
func (this *Console) Log(args ...interface{}) {
|
||||
for index, arg := range args {
|
||||
if arg != nil {
|
||||
switch arg.(type) {
|
||||
case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, string:
|
||||
default:
|
||||
var argType = reflect.TypeOf(arg)
|
||||
|
||||
// 是否有String()方法,如果有直接调用
|
||||
method, ok := argType.MethodByName("String")
|
||||
if ok && method.Type.NumIn() == 1 && method.Type.NumOut() == 1 && method.Type.Out(0).Kind() == reflect.String {
|
||||
args[index] = method.Func.Call([]reflect.Value{reflect.ValueOf(arg)})[0].String()
|
||||
continue
|
||||
}
|
||||
|
||||
// 转为JSON
|
||||
argJSON, err := this.toJSON(arg)
|
||||
if err != nil {
|
||||
if argType.Kind() == reflect.Func {
|
||||
args[index] = "[function]"
|
||||
} else {
|
||||
args[index] = "[object]"
|
||||
}
|
||||
} else {
|
||||
args[index] = string(argJSON)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
args[index] = "null"
|
||||
}
|
||||
}
|
||||
logs.Println(append([]interface{}{"[js][console]"}, args...)...)
|
||||
}
|
||||
|
||||
func (this *Console) toJSON(o interface{}) ([]byte, error) {
|
||||
return json.Marshal(o)
|
||||
}
|
||||
38
internal/js/console_test.go
Normal file
38
internal/js/console_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConsole_Log(t *testing.T) {
|
||||
{
|
||||
vm := NewVM()
|
||||
_, err := vm.RunString("console.log('Hello', 'world')")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
{
|
||||
vm := NewVM()
|
||||
_, err := vm.RunString("console.log(null, true, false, 10, 10.123)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
{
|
||||
vm := NewVM()
|
||||
_, err := vm.RunString("console.log({ a:1, b:2 })")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
{
|
||||
vm := NewVM()
|
||||
_, err := vm.RunString("console.log(console.log)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
36
internal/js/http.go
Normal file
36
internal/js/http.go
Normal file
@@ -0,0 +1,36 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
type HTTP struct {
|
||||
r RequestInterface
|
||||
|
||||
req *Request
|
||||
resp *Response
|
||||
|
||||
onRequest func(req *Request, resp *Response)
|
||||
}
|
||||
|
||||
func NewHTTP(r RequestInterface) *HTTP {
|
||||
return &HTTP{
|
||||
req: NewRequest(r),
|
||||
resp: NewResponse(r),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *HTTP) OnRequest(callback func(req *Request, resp *Response)) {
|
||||
// TODO 考虑是否支持多个callback
|
||||
this.onRequest = callback
|
||||
}
|
||||
|
||||
func (this *HTTP) OnData(callback func(req *Request, resp *Response)) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
func (this *HTTP) OnResponse(callback func(req *Request, resp *Response)) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
func (this *HTTP) TriggerRequest() {
|
||||
this.onRequest(this.req, this.resp)
|
||||
}
|
||||
82
internal/js/request.go
Normal file
82
internal/js/request.go
Normal file
@@ -0,0 +1,82 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
r RequestInterface
|
||||
}
|
||||
|
||||
func NewRequest(r RequestInterface) *Request {
|
||||
return &Request{
|
||||
r: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Request) Proto() string {
|
||||
return this.r.JSRequest().Proto
|
||||
}
|
||||
|
||||
func (this *Request) Method() string {
|
||||
return this.r.JSRequest().Method
|
||||
}
|
||||
|
||||
func (this *Request) Header() map[string][]string {
|
||||
return this.r.JSRequest().Header
|
||||
}
|
||||
|
||||
func (this *Request) AddHeader(name string, value string) {
|
||||
this.r.JSRequest().Header[name] = append(this.r.JSRequest().Header[name], value)
|
||||
}
|
||||
|
||||
func (this *Request) SetHeader(name string, value string) {
|
||||
this.r.JSRequest().Header[name] = []string{value}
|
||||
}
|
||||
|
||||
func (this *Request) RemoteAddr() string {
|
||||
var remoteAddr = this.r.JSRequest().RemoteAddr
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err == nil {
|
||||
return host
|
||||
}
|
||||
return remoteAddr
|
||||
}
|
||||
|
||||
func (this *Request) Url() *URL {
|
||||
return NewURL(this.r.JSRequest().URL)
|
||||
}
|
||||
|
||||
func (this *Request) ContentLength() int64 {
|
||||
return this.r.JSRequest().ContentLength
|
||||
}
|
||||
|
||||
func (this *Request) Body() []byte {
|
||||
var bodyReader = this.r.JSRequest().Body
|
||||
if bodyReader == nil {
|
||||
return []byte{}
|
||||
}
|
||||
data, err := ioutil.ReadAll(bodyReader)
|
||||
if err != nil {
|
||||
this.r.JSLog("read body failed: " + err.Error())
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func (this *Request) CopyBody() []byte {
|
||||
var bodyReader = this.r.JSRequest().Body
|
||||
if bodyReader == nil {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
data, err := ioutil.ReadAll(bodyReader)
|
||||
if err != nil {
|
||||
this.r.JSLog("read body failed: " + err.Error())
|
||||
}
|
||||
this.r.JSRequest().Body = ioutil.NopCloser(bytes.NewReader(data))
|
||||
return data
|
||||
}
|
||||
19
internal/js/request_interface.go
Normal file
19
internal/js/request_interface.go
Normal file
@@ -0,0 +1,19 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import "net/http"
|
||||
|
||||
type RequestInterface interface {
|
||||
// JSRequest 请求
|
||||
JSRequest() *http.Request
|
||||
|
||||
// JSWriter 响应
|
||||
JSWriter() http.ResponseWriter
|
||||
|
||||
// JSStop 中止请求
|
||||
JSStop()
|
||||
|
||||
// JSLog 打印日志
|
||||
JSLog(msg ...interface{})
|
||||
}
|
||||
124
internal/js/request_test.go
Normal file
124
internal/js/request_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/js"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testRequest struct {
|
||||
rawRequest *http.Request
|
||||
rawResponse *testResponse
|
||||
}
|
||||
|
||||
func (this *testRequest) JSRequest() *http.Request {
|
||||
if this.rawRequest != nil {
|
||||
return this.rawRequest
|
||||
}
|
||||
req, _ := http.NewRequest(http.MethodGet, "https://iwind:123456@goedge.cn/docs?name=Libai&age=20", nil)
|
||||
req.Header.Set("Server", "edgejs/1.0")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader([]byte("123456")))
|
||||
this.rawRequest = req
|
||||
return req
|
||||
}
|
||||
|
||||
func (this *testRequest) JSWriter() http.ResponseWriter {
|
||||
if this.rawResponse != nil {
|
||||
return this.rawResponse
|
||||
}
|
||||
this.rawResponse = &testResponse{}
|
||||
return this.rawResponse
|
||||
}
|
||||
|
||||
func (this *testRequest) JSStop() {
|
||||
|
||||
}
|
||||
|
||||
func (this *testRequest) JSLog(msg ...interface{}) {
|
||||
logs.Println(msg...)
|
||||
}
|
||||
|
||||
type testResponse struct {
|
||||
statusCode int
|
||||
header http.Header
|
||||
}
|
||||
|
||||
func (this *testResponse) Header() http.Header {
|
||||
if this.header == nil {
|
||||
this.header = http.Header{}
|
||||
}
|
||||
return this.header
|
||||
}
|
||||
|
||||
func (this *testResponse) Write(p []byte) (int, error) {
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (this *testResponse) WriteHeader(statusCode int) {
|
||||
this.statusCode = statusCode
|
||||
}
|
||||
|
||||
func TestRequest(t *testing.T) {
|
||||
vm := js.NewVM()
|
||||
vm.SetRequest(&testRequest{})
|
||||
|
||||
// 事件监听
|
||||
_, err := vm.RunString(`
|
||||
http.onRequest(function (req, resp) {
|
||||
console.log(req.proto())
|
||||
|
||||
let url = req.url()
|
||||
console.log(url, "port:", url.port(), "args:", url.args())
|
||||
console.log("username:", url.username(), "password:", url.password())
|
||||
console.log("uri:", url.uri(), "path:", url.path())
|
||||
|
||||
req.addHeader("Server", "1.0")
|
||||
|
||||
|
||||
resp.write("this is response")
|
||||
console.log(resp)
|
||||
|
||||
console.log(req.body())
|
||||
})
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 触发事件
|
||||
_, err = vm.RunString(`http.triggerRequest()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequest_Header(t *testing.T) {
|
||||
var req = js.NewRequest(&testRequest{})
|
||||
logs.PrintAsJSON(req.Header(), t)
|
||||
|
||||
req.AddHeader("Content-Length", "10")
|
||||
req.AddHeader("Vary", "1.0")
|
||||
req.AddHeader("Vary", "2.0")
|
||||
logs.PrintAsJSON(req.Header(), t)
|
||||
|
||||
req.SetHeader("Vary", "3.0")
|
||||
logs.PrintAsJSON(req.Header(), t)
|
||||
}
|
||||
|
||||
func TestRequest_Body(t *testing.T) {
|
||||
var req = js.NewRequest(&testRequest{})
|
||||
t.Log(string(req.Body()))
|
||||
t.Log(string(req.Body()))
|
||||
}
|
||||
|
||||
func TestRequest_CopyBody(t *testing.T) {
|
||||
var req = js.NewRequest(&testRequest{})
|
||||
t.Log(string(req.CopyBody()))
|
||||
t.Log(string(req.CopyBody()))
|
||||
}
|
||||
39
internal/js/response.go
Normal file
39
internal/js/response.go
Normal file
@@ -0,0 +1,39 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
type Response struct {
|
||||
r RequestInterface
|
||||
}
|
||||
|
||||
func NewResponse(r RequestInterface) *Response {
|
||||
return &Response{
|
||||
r: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Response) Write(s string) error {
|
||||
_, err := this.r.JSWriter().Write([]byte(s))
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *Response) Reply(status int) {
|
||||
this.SetStatus(status)
|
||||
this.r.JSStop()
|
||||
}
|
||||
|
||||
func (this *Response) Header() map[string][]string {
|
||||
return this.r.JSWriter().Header()
|
||||
}
|
||||
|
||||
func (this *Response) AddHeader(name string, value string) {
|
||||
this.r.JSWriter().Header()[name] = append(this.r.JSWriter().Header()[name], value)
|
||||
}
|
||||
|
||||
func (this *Response) SetHeader(name string, value string) {
|
||||
this.r.JSWriter().Header()[name] = []string{value}
|
||||
}
|
||||
|
||||
func (this *Response) SetStatus(statusCode int) {
|
||||
this.r.JSWriter().WriteHeader(statusCode)
|
||||
}
|
||||
16
internal/js/response_test.go
Normal file
16
internal/js/response_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/js"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewResponse(t *testing.T) {
|
||||
var resp = js.NewResponse(&testRequest{})
|
||||
resp.AddHeader("Vary", "1.0")
|
||||
resp.AddHeader("Vary", "2.0")
|
||||
resp.SetHeader("Server", "edgejs/1.0")
|
||||
t.Logf("%#v", resp.Header())
|
||||
}
|
||||
90
internal/js/url.go
Normal file
90
internal/js/url.go
Normal file
@@ -0,0 +1,90 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"github.com/dop251/goja"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type URL struct {
|
||||
u *url.URL
|
||||
}
|
||||
|
||||
func NewURL(u *url.URL) *URL {
|
||||
return &URL{
|
||||
u: u,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *URL) JSNew(args []goja.Value) *URL {
|
||||
var urlString = ""
|
||||
if len(args) == 1 {
|
||||
urlString = args[0].String()
|
||||
}
|
||||
u, _ := url.Parse(urlString)
|
||||
if u == nil {
|
||||
u = &url.URL{}
|
||||
}
|
||||
return NewURL(u)
|
||||
}
|
||||
|
||||
func (this *URL) Port() int {
|
||||
return types.Int(this.u.Port())
|
||||
}
|
||||
|
||||
func (this *URL) Args() map[string][]string {
|
||||
return this.u.Query()
|
||||
}
|
||||
|
||||
func (this *URL) Arg(name string) string {
|
||||
return this.u.Query().Get(name)
|
||||
}
|
||||
|
||||
func (this *URL) Username() string {
|
||||
if this.u.User != nil {
|
||||
return this.u.User.Username()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (this *URL) Password() string {
|
||||
if this.u.User != nil {
|
||||
password, _ := this.u.User.Password()
|
||||
return password
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (this *URL) Uri() string {
|
||||
return this.u.RequestURI()
|
||||
}
|
||||
|
||||
func (this *URL) Path() string {
|
||||
return this.u.Path
|
||||
}
|
||||
|
||||
func (this *URL) Host() string {
|
||||
return this.u.Host
|
||||
}
|
||||
|
||||
func (this *URL) Fragment() string {
|
||||
return this.u.Fragment
|
||||
}
|
||||
|
||||
func (this *URL) Hash() string {
|
||||
if len(this.u.Fragment) > 0 {
|
||||
return "#" + this.u.Fragment
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (this *URL) Scheme() string {
|
||||
return this.u.Scheme
|
||||
}
|
||||
|
||||
func (this *URL) String() string {
|
||||
return this.u.String()
|
||||
}
|
||||
18
internal/js/url_test.go
Normal file
18
internal/js/url_test.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestURL(t *testing.T) {
|
||||
raw, err := url.Parse("https://iwind:123456@goedge.cn/docs?name=Libai&age=20#a=b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var u = NewURL(raw)
|
||||
t.Log("host:", u.Host())
|
||||
t.Log("hash:", u.Hash())
|
||||
}
|
||||
153
internal/js/vm.go
Normal file
153
internal/js/vm.go
Normal file
@@ -0,0 +1,153 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var sharedPrograms []*goja.Program
|
||||
var sharedConsole = &Console{}
|
||||
|
||||
func init() {
|
||||
// compile programs
|
||||
}
|
||||
|
||||
type VM struct {
|
||||
vm *goja.Runtime
|
||||
}
|
||||
|
||||
func NewVM() *VM {
|
||||
vm := goja.New()
|
||||
vm.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
|
||||
|
||||
// programs
|
||||
for _, program := range sharedPrograms {
|
||||
_, _ = vm.RunProgram(program)
|
||||
}
|
||||
|
||||
v := &VM{vm: vm}
|
||||
v.initVM()
|
||||
return v
|
||||
}
|
||||
|
||||
func (this *VM) Set(name string, obj interface{}) error {
|
||||
return this.vm.Set(name, obj)
|
||||
}
|
||||
|
||||
func (this *VM) AddConstructor(name string, instance interface{}) error {
|
||||
objType := reflect.TypeOf(instance)
|
||||
|
||||
if objType.Kind() != reflect.Ptr {
|
||||
return errors.New("instance should be pointer")
|
||||
}
|
||||
|
||||
// construct
|
||||
newMethod, ok := objType.MethodByName("JSNew")
|
||||
if !ok {
|
||||
return errors.New("can not find 'JSNew()' method in '" + objType.Elem().Name() + "'")
|
||||
}
|
||||
|
||||
var err = this.Set(name, func(call goja.ConstructorCall) *goja.Object {
|
||||
if newMethod.Type.NumIn() != 2 {
|
||||
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should accept a '[]goja.Value' argument"))
|
||||
return nil
|
||||
}
|
||||
if newMethod.Type.In(1).String() != "[]goja.Value" {
|
||||
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should accept a '[]goja.Value' argument"))
|
||||
return nil
|
||||
}
|
||||
|
||||
// new
|
||||
var results = newMethod.Func.Call([]reflect.Value{reflect.ValueOf(instance), reflect.ValueOf(call.Arguments)})
|
||||
if len(results) == 0 {
|
||||
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should return a valid instance"))
|
||||
return nil
|
||||
}
|
||||
var result = results[0]
|
||||
if result.Type() != objType {
|
||||
this.throw(errors.New(objType.Elem().Name() + ".JSNew() should return a same instance"))
|
||||
return nil
|
||||
}
|
||||
|
||||
// methods
|
||||
var resultType = result.Type()
|
||||
var numMethod = result.NumMethod()
|
||||
for i := 0; i < numMethod; i++ {
|
||||
var method = resultType.Method(i)
|
||||
var methodName = strings.ToLower(method.Name[:1]) + method.Name[1:]
|
||||
err := call.This.Set(methodName, result.MethodByName(method.Name).Interface())
|
||||
if err != nil {
|
||||
this.throw(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 支持属性
|
||||
var numField = result.Elem().Type().NumField()
|
||||
for i := 0; i < numField; i++ {
|
||||
var field = result.Elem().Field(i)
|
||||
if !field.CanInterface() {
|
||||
continue
|
||||
}
|
||||
var fieldType = objType.Elem().Field(i)
|
||||
tag, ok := fieldType.Tag.Lookup("json")
|
||||
if !ok {
|
||||
tag = fieldType.Name
|
||||
tag = strings.ToLower(tag[:1]) + tag[1:]
|
||||
} else {
|
||||
// TODO 校验tag是否符合变量语法
|
||||
}
|
||||
err := call.This.Set(tag, field.Interface())
|
||||
if err != nil {
|
||||
this.throw(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *VM) RunString(str string) (goja.Value, error) {
|
||||
defer func() {
|
||||
e := recover()
|
||||
if e != nil {
|
||||
// TODO 需要打印trace
|
||||
logs.Println("panic:", e)
|
||||
}
|
||||
}()
|
||||
return this.vm.RunString(str)
|
||||
}
|
||||
|
||||
func (this *VM) SetRequest(req RequestInterface) {
|
||||
{
|
||||
err := this.vm.Set("http", NewHTTP(req))
|
||||
if err != nil {
|
||||
this.throw(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *VM) initVM() {
|
||||
{
|
||||
err := this.vm.Set("console", sharedConsole)
|
||||
if err != nil {
|
||||
this.throw(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *VM) throw(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// TODO
|
||||
logs.Println("js:VM:error: " + err.Error())
|
||||
}
|
||||
158
internal/js/vm_test.go
Normal file
158
internal/js/vm_test.go
Normal file
@@ -0,0 +1,158 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"github.com/dop251/goja"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewVM(t *testing.T) {
|
||||
before := time.Now()
|
||||
defer func() {
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
vm := NewVM()
|
||||
{
|
||||
v, err := vm.RunString("JSON.stringify({\"a\":\"b\"})")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("JSON.stringify():", v)
|
||||
}
|
||||
{
|
||||
v, err := vm.RunString(`JSON.parse('{\"a\":\"b\"}')`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("JSON.parse():", v)
|
||||
}
|
||||
{
|
||||
err := vm.AddConstructor("Url", &URL{})
|
||||
if err != nil {
|
||||
t.Fatal("add constructor error:", err)
|
||||
}
|
||||
_, err = vm.RunString(`
|
||||
{
|
||||
let u = new Url("https://goedge.cn/docs?v=1")
|
||||
console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
{
|
||||
let u = new Url("https://teaos.cn/downloads?v=1")
|
||||
console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url()
|
||||
console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url("a", "b", "c")
|
||||
console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatal("add constructor error:" + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVM_Program(t *testing.T) {
|
||||
var s = `
|
||||
{
|
||||
let u = new Url("https://goedge.cn/docs?v=1")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
{
|
||||
let u = new Url("https://teaos.cn/downloads?v=1")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url()
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url("a", "b", "c")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
`
|
||||
program := goja.MustCompile("s", s, true)
|
||||
|
||||
before := time.Now()
|
||||
defer func() {
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
vm := NewVM()
|
||||
err := vm.AddConstructor("Url", &URL{})
|
||||
if err != nil {
|
||||
t.Fatal("add constructor error:", err)
|
||||
}
|
||||
//_, err = vm.RunString(s)
|
||||
_, err = vm.vm.RunProgram(program)
|
||||
if err != nil {
|
||||
t.Fatal("add constructor error:" + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark_Program(b *testing.B) {
|
||||
var s = `
|
||||
{
|
||||
let u = new Url("https://goedge.cn/docs?v=1")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
{
|
||||
let u = new Url("https://teaos.cn/downloads?v=1")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url()
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url("a", "b", "c")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
{
|
||||
let u = new Url("https://goedge.cn/docs?v=1")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
{
|
||||
let u = new Url("https://teaos.cn/downloads?v=1")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url()
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
|
||||
{
|
||||
let u = new Url("a", "b", "c")
|
||||
//console.log("host:", u.host(), u.uri())
|
||||
}
|
||||
`
|
||||
program := goja.MustCompile("s", s, true)
|
||||
|
||||
vm := NewVM()
|
||||
|
||||
err := vm.AddConstructor("Url", &URL{})
|
||||
if err != nil {
|
||||
b.Fatal("add constructor error:", err)
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
//_, err = vm.RunString(s)
|
||||
_, err = vm.vm.RunProgram(program)
|
||||
if err != nil {
|
||||
b.Fatal("add constructor error:" + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
121
internal/metrics/manager.go
Normal file
121
internal/metrics/manager.go
Normal file
@@ -0,0 +1,121 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var SharedManager = NewManager()
|
||||
|
||||
type Manager struct {
|
||||
tasks map[int64]*Task // itemId => *Task
|
||||
categoryTasks map[string][]*Task // category => []*Task
|
||||
locker sync.RWMutex
|
||||
|
||||
hasHTTPMetrics bool
|
||||
hasTCPMetrics bool
|
||||
hasUDPMetrics bool
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
return &Manager{
|
||||
tasks: map[int64]*Task{},
|
||||
categoryTasks: map[string][]*Task{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var newMap = map[int64]*serverconfigs.MetricItemConfig{}
|
||||
for _, item := range items {
|
||||
newMap[item.Id] = item
|
||||
}
|
||||
|
||||
// 停用以前的 或 修改现在的
|
||||
for itemId, task := range this.tasks {
|
||||
newItem, ok := newMap[itemId]
|
||||
if !ok || !newItem.IsOn { // 停用以前的
|
||||
remotelogs.Println("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"'")
|
||||
err := task.Stop()
|
||||
if err != nil {
|
||||
remotelogs.Error("METRIC_MANAGER", "stop task '"+strconv.FormatInt(itemId, 10)+"' failed: "+err.Error())
|
||||
}
|
||||
delete(this.tasks, itemId)
|
||||
} else { // 更新已存在的
|
||||
if newItem.Version != task.item.Version {
|
||||
remotelogs.Println("METRIC_MANAGER", "update task '"+strconv.FormatInt(itemId, 10)+"'")
|
||||
task.item = newItem
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 启动新的
|
||||
for _, newItem := range items {
|
||||
if !newItem.IsOn {
|
||||
continue
|
||||
}
|
||||
_, ok := this.tasks[newItem.Id]
|
||||
if !ok {
|
||||
remotelogs.Println("METRIC_MANAGER", "start task '"+strconv.FormatInt(newItem.Id, 10)+"'")
|
||||
task := NewTask(newItem)
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
remotelogs.Error("METRIC_MANAGER", "initialized task failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
err = task.Start()
|
||||
if err != nil {
|
||||
remotelogs.Error("METRIC_MANAGER", "start task failed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
this.tasks[newItem.Id] = task
|
||||
}
|
||||
}
|
||||
|
||||
// 按分类存放
|
||||
this.hasHTTPMetrics = false
|
||||
this.hasTCPMetrics = false
|
||||
this.hasUDPMetrics = false
|
||||
this.categoryTasks = map[string][]*Task{}
|
||||
for _, task := range this.tasks {
|
||||
tasks := this.categoryTasks[task.item.Category]
|
||||
tasks = append(tasks, task)
|
||||
this.categoryTasks[task.item.Category] = tasks
|
||||
|
||||
switch task.item.Category {
|
||||
case serverconfigs.MetricItemCategoryHTTP:
|
||||
this.hasHTTPMetrics = true
|
||||
case serverconfigs.MetricItemCategoryTCP:
|
||||
this.hasTCPMetrics = true
|
||||
case serverconfigs.MetricItemCategoryUDP:
|
||||
this.hasUDPMetrics = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add 添加数据
|
||||
func (this *Manager) Add(obj MetricInterface) {
|
||||
this.locker.RLock()
|
||||
for _, task := range this.categoryTasks[obj.MetricCategory()] {
|
||||
task.Add(obj)
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
}
|
||||
|
||||
func (this *Manager) HasHTTPMetrics() bool {
|
||||
return this.hasHTTPMetrics
|
||||
}
|
||||
|
||||
func (this *Manager) HasTCPMetrics() bool {
|
||||
return this.hasTCPMetrics
|
||||
}
|
||||
|
||||
func (this *Manager) HasUDPMetrics() bool {
|
||||
return this.hasUDPMetrics
|
||||
}
|
||||
63
internal/metrics/manager_test.go
Normal file
63
internal/metrics/manager_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewManager(t *testing.T) {
|
||||
var manager = NewManager()
|
||||
{
|
||||
manager.Update([]*serverconfigs.MetricItemConfig{})
|
||||
for _, task := range manager.tasks {
|
||||
t.Log(task.item.Id)
|
||||
}
|
||||
}
|
||||
{
|
||||
t.Log("====")
|
||||
manager.Update([]*serverconfigs.MetricItemConfig{
|
||||
{
|
||||
Id: 1,
|
||||
},
|
||||
{
|
||||
Id: 2,
|
||||
},
|
||||
{
|
||||
Id: 3,
|
||||
},
|
||||
})
|
||||
for _, task := range manager.tasks {
|
||||
t.Log("task:", task.item.Id)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
t.Log("====")
|
||||
manager.Update([]*serverconfigs.MetricItemConfig{
|
||||
{
|
||||
Id: 1,
|
||||
},
|
||||
{
|
||||
Id: 2,
|
||||
},
|
||||
})
|
||||
for _, task := range manager.tasks {
|
||||
t.Log("task:", task.item.Id)
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
t.Log("====")
|
||||
manager.Update([]*serverconfigs.MetricItemConfig{
|
||||
{
|
||||
Id: 1,
|
||||
Version: 1,
|
||||
},
|
||||
})
|
||||
for _, task := range manager.tasks {
|
||||
t.Log("task:", task.item.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
17
internal/metrics/metric_interface.go
Normal file
17
internal/metrics/metric_interface.go
Normal file
@@ -0,0 +1,17 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package metrics
|
||||
|
||||
type MetricInterface interface {
|
||||
// MetricKey 指标对象
|
||||
MetricKey(key string) string
|
||||
|
||||
// MetricValue 指标值
|
||||
MetricValue(value string) (result int64, ok bool)
|
||||
|
||||
// MetricServerId 服务ID
|
||||
MetricServerId() int64
|
||||
|
||||
// MetricCategory 指标分类
|
||||
MetricCategory() string
|
||||
}
|
||||
22
internal/metrics/stat.go
Normal file
22
internal/metrics/stat.go
Normal file
@@ -0,0 +1,22 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/cespare/xxhash"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Stat struct {
|
||||
ServerId int64
|
||||
Keys []string
|
||||
Hash string
|
||||
Value int64
|
||||
Time string
|
||||
}
|
||||
|
||||
func SumStat(serverId int64, keys []string, time string, version int32, itemId int64) string {
|
||||
keysData, _ := json.Marshal(keys)
|
||||
return strconv.FormatUint(xxhash.Sum64String(strconv.FormatInt(serverId, 10)+"@"+string(keysData)+"@"+time+"@"+strconv.Itoa(int(version))+"@"+strconv.FormatInt(itemId, 10)), 10)
|
||||
}
|
||||
509
internal/metrics/task.go
Normal file
509
internal/metrics/task.go
Normal file
@@ -0,0 +1,509 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const MaxQueueSize = 10240
|
||||
|
||||
// Task 单个指标任务
|
||||
// 数据库存储:
|
||||
// data/
|
||||
// metric.$ID.db
|
||||
// stats
|
||||
// id, keys, value, time, serverId, hash
|
||||
// 原理:
|
||||
// 添加或者有变更时 isUploaded = false
|
||||
// 上传时检查 isUploaded 状态
|
||||
// 只上传每个服务中排序最前面的 N 个数据
|
||||
type Task struct {
|
||||
item *serverconfigs.MetricItemConfig
|
||||
isLoaded bool
|
||||
|
||||
db *sql.DB
|
||||
statTableName string
|
||||
isStopped bool
|
||||
|
||||
cleanTicker *utils.Ticker
|
||||
uploadTicker *utils.Ticker
|
||||
|
||||
cleanVersion int32
|
||||
|
||||
insertStatStmt *sql.Stmt
|
||||
deleteByVersionStmt *sql.Stmt
|
||||
deleteByExpiresTimeStmt *sql.Stmt
|
||||
selectTopStmt *sql.Stmt
|
||||
sumStmt *sql.Stmt
|
||||
|
||||
serverIdMap map[int64]bool // 所有的服务Ids
|
||||
timeMap map[string]bool // time => bool
|
||||
serverIdMapLocker sync.Mutex
|
||||
|
||||
statsMap map[string]*Stat
|
||||
statsLocker sync.Mutex
|
||||
statsTicker *utils.Ticker
|
||||
}
|
||||
|
||||
// NewTask 获取新任务
|
||||
func NewTask(item *serverconfigs.MetricItemConfig) *Task {
|
||||
return &Task{
|
||||
item: item,
|
||||
serverIdMap: map[int64]bool{},
|
||||
timeMap: map[string]bool{},
|
||||
statsMap: map[string]*Stat{},
|
||||
}
|
||||
}
|
||||
|
||||
// Init 初始化
|
||||
func (this *Task) Init() error {
|
||||
this.statTableName = "stats"
|
||||
|
||||
// 检查目录是否存在
|
||||
var dir = Tea.Root + "/data"
|
||||
_, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
err = os.MkdirAll(dir, 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
this.db = db
|
||||
|
||||
//创建统计表
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.statTableName + `" (
|
||||
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||
"hash" varchar(32),
|
||||
"keys" varchar(1024),
|
||||
"value" real DEFAULT 0,
|
||||
"time" varchar(32),
|
||||
"serverId" integer DEFAULT 0,
|
||||
"version" integer DEFAULT 0,
|
||||
"isUploaded" integer DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "serverId"
|
||||
ON "` + this.statTableName + `" (
|
||||
"serverId" ASC,
|
||||
"version" ASC
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
|
||||
ON "` + this.statTableName + `" (
|
||||
"hash" ASC
|
||||
);`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert stat stmt
|
||||
this.insertStatStmt, err = db.Prepare(`INSERT INTO "stats" ("serverId", "hash", "keys", "value", "time", "version", "isUploaded") VALUES (?, ?, ?, ?, ?, ?, 0) ON CONFLICT("hash") DO UPDATE SET "value"="value"+?, "isUploaded"=0`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete by version
|
||||
this.deleteByVersionStmt, err = db.Prepare(`DELETE FROM "` + this.statTableName + `" WHERE "version"<?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete by expires time
|
||||
this.deleteByExpiresTimeStmt, err = db.Prepare(`DELETE FROM "` + this.statTableName + `" WHERE "time"<?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// select topN stmt
|
||||
this.selectTopStmt, err = db.Prepare(`SELECT "id", "hash", "keys", "value", "isUploaded" FROM "` + this.statTableName + `" WHERE "serverId"=? AND "version"=? AND time=? ORDER BY "value" DESC LIMIT 20`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// sum stmt
|
||||
this.sumStmt, err = db.Prepare(`SELECT COUNT(*), IFNULL(SUM(value), 0) FROM "` + this.statTableName + `" WHERE "serverId"=? AND "version"=? AND time=?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 所有的服务IDs
|
||||
err = this.loadServerIdMap()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.isLoaded = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start 启动任务
|
||||
func (this *Task) Start() error {
|
||||
// 读取数据
|
||||
this.statsTicker = utils.NewTicker(1 * time.Minute)
|
||||
go func() {
|
||||
for this.statsTicker.Next() {
|
||||
this.statsLocker.Lock()
|
||||
var statsMap = this.statsMap
|
||||
this.statsMap = map[string]*Stat{}
|
||||
this.statsLocker.Unlock()
|
||||
|
||||
for _, stat := range statsMap {
|
||||
err := this.InsertStat(stat)
|
||||
if err != nil {
|
||||
remotelogs.Error("METRIC", "insert stat failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// 清理
|
||||
this.cleanTicker = utils.NewTicker(24 * time.Hour)
|
||||
go func() {
|
||||
for this.cleanTicker.Next() {
|
||||
err := this.CleanExpired()
|
||||
if err != nil {
|
||||
remotelogs.Error("METRIC", "clean expired stats failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// 上传
|
||||
this.uploadTicker = utils.NewTicker(this.item.UploadDuration())
|
||||
go func() {
|
||||
for this.uploadTicker.Next() {
|
||||
err := this.Upload(1 * time.Second)
|
||||
if err != nil {
|
||||
remotelogs.Error("METRIC", "upload stats failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add 添加数据
|
||||
func (this *Task) Add(obj MetricInterface) {
|
||||
if this.isStopped || !this.isLoaded {
|
||||
return
|
||||
}
|
||||
|
||||
var keys = []string{}
|
||||
for _, key := range this.item.Keys {
|
||||
k := obj.MetricKey(key)
|
||||
|
||||
// 忽略499状态
|
||||
if key == "${status}" && k == "499" {
|
||||
return
|
||||
}
|
||||
|
||||
keys = append(keys, k)
|
||||
}
|
||||
|
||||
v, ok := obj.MetricValue(this.item.Value)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var hash = SumStat(obj.MetricServerId(), keys, this.item.CurrentTime(), this.item.Version, this.item.Id)
|
||||
this.statsLocker.Lock()
|
||||
oldStat, ok := this.statsMap[hash]
|
||||
if ok {
|
||||
oldStat.Value += v
|
||||
oldStat.Hash = hash
|
||||
} else {
|
||||
// 防止过载
|
||||
if len(this.statsMap) < MaxQueueSize {
|
||||
this.statsMap[hash] = &Stat{
|
||||
ServerId: obj.MetricServerId(),
|
||||
Keys: keys,
|
||||
Value: v,
|
||||
Time: this.item.CurrentTime(),
|
||||
Hash: hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
this.statsLocker.Unlock()
|
||||
}
|
||||
|
||||
// Stop 停止任务
|
||||
func (this *Task) Stop() error {
|
||||
this.isStopped = true
|
||||
|
||||
if this.cleanTicker != nil {
|
||||
this.cleanTicker.Stop()
|
||||
}
|
||||
if this.uploadTicker != nil {
|
||||
this.uploadTicker.Stop()
|
||||
}
|
||||
if this.statsTicker != nil {
|
||||
this.statsTicker.Stop()
|
||||
}
|
||||
|
||||
_ = this.insertStatStmt.Close()
|
||||
_ = this.deleteByVersionStmt.Close()
|
||||
_ = this.deleteByExpiresTimeStmt.Close()
|
||||
_ = this.selectTopStmt.Close()
|
||||
_ = this.sumStmt.Close()
|
||||
|
||||
if this.db != nil {
|
||||
_ = this.db.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertStat 写入数据
|
||||
func (this *Task) InsertStat(stat *Stat) error {
|
||||
if this.isStopped {
|
||||
return nil
|
||||
}
|
||||
if stat == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.serverIdMapLocker.Lock()
|
||||
this.serverIdMap[stat.ServerId] = true
|
||||
this.timeMap[stat.Time] = true
|
||||
this.serverIdMapLocker.Unlock()
|
||||
|
||||
keyData, err := json.Marshal(stat.Keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = this.insertStatStmt.Exec(stat.ServerId, stat.Hash, keyData, stat.Value, stat.Time, this.item.Version, stat.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanExpired 清理数据
|
||||
func (this *Task) CleanExpired() error {
|
||||
if this.isStopped {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 清除低版本数据
|
||||
if this.cleanVersion < this.item.Version {
|
||||
_, err := this.deleteByVersionStmt.Exec(this.item.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.cleanVersion = this.item.Version
|
||||
}
|
||||
|
||||
// 清除过期的数据
|
||||
_, err := this.deleteByExpiresTimeStmt.Exec(this.item.LocalExpiresTime())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Upload 上传数据
|
||||
func (this *Task) Upload(pauseDuration time.Duration) error {
|
||||
if this.isStopped {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.serverIdMapLocker.Lock()
|
||||
|
||||
// 服务IDs
|
||||
var serverIds []int64
|
||||
for serverId := range this.serverIdMap {
|
||||
serverIds = append(serverIds, serverId)
|
||||
}
|
||||
this.serverIdMap = map[int64]bool{} // 清空数据
|
||||
|
||||
// 时间
|
||||
var times = []string{}
|
||||
for t := range this.timeMap {
|
||||
times = append(times, t)
|
||||
}
|
||||
this.timeMap = map[string]bool{} // 清空数据
|
||||
|
||||
this.serverIdMapLocker.Unlock()
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, serverId := range serverIds {
|
||||
for _, currentTime := range times {
|
||||
idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
|
||||
rows, err := this.selectTopStmt.Query(serverId, this.item.Version, currentTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var isClosed bool
|
||||
defer func() {
|
||||
if isClosed {
|
||||
return
|
||||
}
|
||||
_ = rows.Close()
|
||||
}()
|
||||
|
||||
var pbStats []*pb.UploadingMetricStat
|
||||
for rows.Next() {
|
||||
var pbStat = &pb.UploadingMetricStat{}
|
||||
// "id", "hash", "keys", "value", "isUploaded"
|
||||
var isUploaded int
|
||||
var keysData []byte
|
||||
err = rows.Scan(&pbStat.Id, &pbStat.Hash, &keysData, &pbStat.Value, &isUploaded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// TODO 先不判断是否已经上传,需要改造API进行配合
|
||||
/**if isUploaded == 1 {
|
||||
continue
|
||||
}**/
|
||||
if len(keysData) > 0 {
|
||||
err = json.Unmarshal(keysData, &pbStat.Keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
pbStats = append(pbStats, pbStat)
|
||||
ids = append(ids, strconv.FormatInt(pbStat.Id, 10))
|
||||
}
|
||||
|
||||
// 提前关闭
|
||||
_ = rows.Close()
|
||||
isClosed = true
|
||||
|
||||
// 上传
|
||||
if len(pbStats) > 0 {
|
||||
// 计算总和
|
||||
count, total, err := this.sum(serverId, currentTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
|
||||
MetricStats: pbStats,
|
||||
Time: currentTime,
|
||||
ServerId: serverId,
|
||||
ItemId: this.item.Id,
|
||||
Version: this.item.Version,
|
||||
Count: count,
|
||||
Total: float32(total),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}(serverId, currentTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(idStrings) > 0 {
|
||||
// 设置为已上传
|
||||
_, err = this.db.Exec(`UPDATE "` + this.statTableName + `" SET isUploaded=1 WHERE id IN (` + strings.Join(idStrings, ",") + `)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 休息一下,防止短时间内上传数据过多
|
||||
if pauseDuration > 0 {
|
||||
time.Sleep(pauseDuration)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 加载服务ID
|
||||
func (this *Task) loadServerIdMap() error {
|
||||
{
|
||||
rows, err := this.db.Query(`SELECT DISTINCT "serverId" FROM `+this.statTableName+" WHERE version=?", this.item.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
|
||||
var serverId int64
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&serverId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.serverIdMapLocker.Lock()
|
||||
this.serverIdMap[serverId] = true
|
||||
this.serverIdMapLocker.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
rows, err := this.db.Query(`SELECT DISTINCT "time" FROM `+this.statTableName+" WHERE version=?", this.item.Version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
|
||||
var timeString string
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&timeString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.serverIdMapLocker.Lock()
|
||||
this.timeMap[timeString] = true
|
||||
this.serverIdMapLocker.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 计算数量和综合
|
||||
func (this *Task) sum(serverId int64, time string) (count int64, total float64, err error) {
|
||||
rows, err := this.sumStmt.Query(serverId, this.item.Version, time)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
}()
|
||||
if rows.Next() {
|
||||
err = rows.Scan(&count, &total)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
210
internal/metrics/task_test.go
Normal file
210
internal/metrics/task_test.go
Normal file
@@ -0,0 +1,210 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package metrics_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/metrics"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type testObj struct {
|
||||
ip string
|
||||
}
|
||||
|
||||
func (this *testObj) MetricKey(key string) string {
|
||||
return this.ip
|
||||
}
|
||||
|
||||
func (this *testObj) MetricValue(value string) (int64, bool) {
|
||||
return 1, true
|
||||
}
|
||||
|
||||
func (this *testObj) MetricServerId() int64 {
|
||||
return int64(rands.Int(1, 100))
|
||||
}
|
||||
|
||||
func (this *testObj) MetricCategory() string {
|
||||
return "http"
|
||||
}
|
||||
|
||||
func TestTask_Init(t *testing.T) {
|
||||
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
|
||||
Id: 1,
|
||||
IsOn: false,
|
||||
Category: "",
|
||||
Period: 0,
|
||||
PeriodUnit: "",
|
||||
Keys: nil,
|
||||
Value: "",
|
||||
})
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = task.Stop()
|
||||
}()
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestTask_Add(t *testing.T) {
|
||||
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
|
||||
Id: 1,
|
||||
IsOn: false,
|
||||
Category: "",
|
||||
Period: 1,
|
||||
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
|
||||
Keys: []string{"${remoteAddr}"},
|
||||
Value: "${countRequest}",
|
||||
})
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = task.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = task.Stop()
|
||||
}()
|
||||
|
||||
task.Add(&testObj{ip: "127.0.0.2"})
|
||||
time.Sleep(1 * time.Second) // waiting for inserting
|
||||
}
|
||||
|
||||
func TestTask_Add_Many(t *testing.T) {
|
||||
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
|
||||
Id: 1,
|
||||
IsOn: false,
|
||||
Category: "",
|
||||
Period: 1,
|
||||
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
|
||||
Keys: []string{"${remoteAddr}"},
|
||||
Value: "${countRequest}",
|
||||
Version: 1,
|
||||
})
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = task.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = task.Stop()
|
||||
}()
|
||||
|
||||
for i := 0; i < 4_000_000; i++ {
|
||||
task.Add(&testObj{
|
||||
ip: fmt.Sprintf("%d.%d.%d.%d", rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255)),
|
||||
})
|
||||
if i%10000 == 0 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTask_InsertStat(t *testing.T) {
|
||||
var item = &serverconfigs.MetricItemConfig{
|
||||
Id: 1,
|
||||
IsOn: false,
|
||||
Category: "",
|
||||
Period: 1,
|
||||
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
|
||||
Keys: []string{"${remoteAddr}"},
|
||||
Value: "${countRequest}",
|
||||
Version: 1,
|
||||
}
|
||||
var task = metrics.NewTask(item)
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = task.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = task.Stop()
|
||||
}()
|
||||
|
||||
err = task.InsertStat(&metrics.Stat{
|
||||
ServerId: 1,
|
||||
Keys: []string{"127.0.0.1"},
|
||||
Hash: "",
|
||||
Value: 1,
|
||||
Time: item.CurrentTime(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestTask_CleanExpired(t *testing.T) {
|
||||
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
|
||||
Id: 1,
|
||||
IsOn: false,
|
||||
Category: "",
|
||||
Period: 1,
|
||||
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
|
||||
Keys: []string{"${remoteAddr}"},
|
||||
Value: "${countRequest}",
|
||||
Version: 1,
|
||||
})
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = task.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = task.Stop()
|
||||
}()
|
||||
|
||||
err = task.CleanExpired()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestTask_Upload(t *testing.T) {
|
||||
var task = metrics.NewTask(&serverconfigs.MetricItemConfig{
|
||||
Id: 1,
|
||||
IsOn: false,
|
||||
Category: "",
|
||||
Period: 1,
|
||||
PeriodUnit: serverconfigs.MetricItemPeriodUnitDay,
|
||||
Keys: []string{"${remoteAddr}"},
|
||||
Value: "${countRequest}",
|
||||
Version: 1,
|
||||
})
|
||||
err := task.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = task.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = task.Stop()
|
||||
}()
|
||||
|
||||
err = task.Upload(0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
var SharedValueQueue = NewValueQueue()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
go SharedValueQueue.Start()
|
||||
})
|
||||
}
|
||||
@@ -72,7 +72,8 @@ func (this *ValueQueue) Loop() error {
|
||||
CreatedAt: value.CreatedAt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
remotelogs.Error("MONITOR", err.Error())
|
||||
continue
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/messageconfigs"
|
||||
@@ -13,7 +15,9 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
@@ -55,10 +59,16 @@ func (this *APIStream) loop() error {
|
||||
return errors.Wrap(err)
|
||||
}
|
||||
isQuiting := false
|
||||
ctx, cancelFunc := context.WithCancel(rpcClient.Context())
|
||||
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
|
||||
events.On(events.EventQuit, func() {
|
||||
isQuiting = true
|
||||
|
||||
remotelogs.Println("API_STREAM", "quiting")
|
||||
if nodeStream != nil {
|
||||
cancelFunc()
|
||||
}
|
||||
})
|
||||
nodeStream, err := rpcClient.NodeRPC().NodeStream(rpcClient.Context())
|
||||
if err != nil {
|
||||
if isQuiting {
|
||||
return nil
|
||||
@@ -69,12 +79,14 @@ func (this *APIStream) loop() error {
|
||||
|
||||
for {
|
||||
if isQuiting {
|
||||
logs.Println("API_STREAM", "quit")
|
||||
break
|
||||
}
|
||||
|
||||
message, err := nodeStream.Recv()
|
||||
if err != nil {
|
||||
if isQuiting {
|
||||
remotelogs.Println("API_STREAM", "quit")
|
||||
return nil
|
||||
}
|
||||
return errors.Wrap(err)
|
||||
@@ -133,6 +145,16 @@ func (this *APIStream) handleConnectedAPINode(message *pb.NodeStreamMessage) err
|
||||
return errors.Wrap(err)
|
||||
}
|
||||
remotelogs.Println("API_STREAM", "connected to api node '"+strconv.FormatInt(msg.APINodeId, 10)+"'")
|
||||
|
||||
// 重新读取配置
|
||||
if nodeConfigUpdatedAt == 0 {
|
||||
select {
|
||||
case nodeConfigChangedNotify <- true:
|
||||
default:
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -320,6 +342,15 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
|
||||
}()
|
||||
}
|
||||
|
||||
// WEBP缓存
|
||||
if msg.Type == "file" {
|
||||
var keys = msg.Keys
|
||||
for _, key := range keys {
|
||||
keys = append(keys, key+webpSuffix)
|
||||
}
|
||||
msg.Keys = keys
|
||||
}
|
||||
|
||||
err = storage.Purge(msg.Keys, msg.Type)
|
||||
if err != nil {
|
||||
this.replyFail(message.RequestId, "purge keys failed: "+err.Error())
|
||||
@@ -357,7 +388,28 @@ func (this *APIStream) handlePreheatCache(message *pb.NodeStreamMessage) error {
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(len(msg.Keys))
|
||||
client := http.Client{} // TODO 可以设置请求超时事件
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second, // TODO 可以设置请求超时时间
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
_, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return net.Dial(network, "127.0.0.1:"+port)
|
||||
},
|
||||
MaxIdleConns: 4096,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
MaxConnsPerHost: 32,
|
||||
IdleConnTimeout: 2 * time.Minute,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSHandshakeTimeout: 0,
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
defer client.CloseIdleConnections()
|
||||
errorMessages := []string{}
|
||||
locker := sync.Mutex{}
|
||||
for _, key := range msg.Keys {
|
||||
@@ -371,7 +423,9 @@ func (this *APIStream) handlePreheatCache(message *pb.NodeStreamMessage) error {
|
||||
locker.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// TODO 可以在管理界面自定义Header
|
||||
req.Header.Set("X-Cache-Action", "preheat")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36")
|
||||
req.Header.Set("Accept-Encoding", "gzip, deflate, br") // TODO 这里需要记录下缓存是否为gzip的
|
||||
resp, err := client.Do(req)
|
||||
|
||||
@@ -42,16 +42,25 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
// TrafficConn 用于统计流量的连接
|
||||
type TrafficConn struct {
|
||||
rawConn net.Conn
|
||||
// ClientConn 客户端连接
|
||||
type ClientConn struct {
|
||||
rawConn net.Conn
|
||||
isClosed bool
|
||||
}
|
||||
|
||||
func NewTrafficConn(conn net.Conn) net.Conn {
|
||||
return &TrafficConn{rawConn: conn}
|
||||
func NewClientConn(conn net.Conn, quickClose bool) net.Conn {
|
||||
if quickClose {
|
||||
tcpConn, ok := conn.(*net.TCPConn)
|
||||
if ok {
|
||||
// TODO 可以设置此值
|
||||
_ = tcpConn.SetLinger(3)
|
||||
}
|
||||
}
|
||||
|
||||
return &ClientConn{rawConn: conn}
|
||||
}
|
||||
|
||||
func (this *TrafficConn) Read(b []byte) (n int, err error) {
|
||||
func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
n, err = this.rawConn.Read(b)
|
||||
if n > 0 {
|
||||
atomic.AddUint64(&inTrafficBytes, uint64(n))
|
||||
@@ -59,7 +68,7 @@ func (this *TrafficConn) Read(b []byte) (n int, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (this *TrafficConn) Write(b []byte) (n int, err error) {
|
||||
func (this *ClientConn) Write(b []byte) (n int, err error) {
|
||||
n, err = this.rawConn.Write(b)
|
||||
if n > 0 {
|
||||
atomic.AddUint64(&outTrafficBytes, uint64(n))
|
||||
@@ -67,26 +76,31 @@ func (this *TrafficConn) Write(b []byte) (n int, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (this *TrafficConn) Close() error {
|
||||
func (this *ClientConn) Close() error {
|
||||
this.isClosed = true
|
||||
return this.rawConn.Close()
|
||||
}
|
||||
|
||||
func (this *TrafficConn) LocalAddr() net.Addr {
|
||||
func (this *ClientConn) LocalAddr() net.Addr {
|
||||
return this.rawConn.LocalAddr()
|
||||
}
|
||||
|
||||
func (this *TrafficConn) RemoteAddr() net.Addr {
|
||||
func (this *ClientConn) RemoteAddr() net.Addr {
|
||||
return this.rawConn.RemoteAddr()
|
||||
}
|
||||
|
||||
func (this *TrafficConn) SetDeadline(t time.Time) error {
|
||||
func (this *ClientConn) SetDeadline(t time.Time) error {
|
||||
return this.rawConn.SetDeadline(t)
|
||||
}
|
||||
|
||||
func (this *TrafficConn) SetReadDeadline(t time.Time) error {
|
||||
func (this *ClientConn) SetReadDeadline(t time.Time) error {
|
||||
return this.rawConn.SetReadDeadline(t)
|
||||
}
|
||||
|
||||
func (this *TrafficConn) SetWriteDeadline(t time.Time) error {
|
||||
func (this *ClientConn) SetWriteDeadline(t time.Time) error {
|
||||
return this.rawConn.SetWriteDeadline(t)
|
||||
}
|
||||
|
||||
func (this *ClientConn) IsClosed() bool {
|
||||
return this.isClosed
|
||||
}
|
||||
22
internal/nodes/client_conn_utils.go
Normal file
22
internal/nodes/client_conn_utils.go
Normal file
@@ -0,0 +1,22 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"net"
|
||||
)
|
||||
|
||||
// 判断客户端连接是否已关闭
|
||||
func isClientConnClosed(conn net.Conn) bool {
|
||||
if conn == nil {
|
||||
return true
|
||||
}
|
||||
clientConn, ok := conn.(*ClientConn)
|
||||
if ok {
|
||||
return clientConn.IsClosed()
|
||||
}
|
||||
|
||||
// TODO 解决tls.Conn无法获取底层连接对象的问题
|
||||
|
||||
return false
|
||||
}
|
||||
54
internal/nodes/client_listener.go
Normal file
54
internal/nodes/client_listener.go
Normal file
@@ -0,0 +1,54 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"net"
|
||||
)
|
||||
|
||||
// ClientListener 客户端网络监听
|
||||
type ClientListener struct {
|
||||
rawListener net.Listener
|
||||
quickClose bool
|
||||
}
|
||||
|
||||
func NewClientListener(listener net.Listener, quickClose bool) net.Listener {
|
||||
return &ClientListener{
|
||||
rawListener: listener,
|
||||
quickClose: quickClose,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ClientListener) Accept() (net.Conn, error) {
|
||||
conn, err := this.rawListener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 是否在WAF名单中
|
||||
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
|
||||
if err == nil {
|
||||
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
|
||||
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
|
||||
|
||||
tcpConn, ok := conn.(*net.TCPConn)
|
||||
if ok {
|
||||
_ = tcpConn.SetLinger(0)
|
||||
}
|
||||
|
||||
_ = conn.Close()
|
||||
return this.Accept()
|
||||
}
|
||||
}
|
||||
|
||||
return NewClientConn(conn, this.quickClose), nil
|
||||
}
|
||||
|
||||
func (this *ClientListener) Close() error {
|
||||
return this.rawListener.Close()
|
||||
}
|
||||
|
||||
func (this *ClientListener) Addr() net.Addr {
|
||||
return this.rawListener.Addr()
|
||||
}
|
||||
@@ -9,14 +9,14 @@ import (
|
||||
|
||||
var sharedHTTPAccessLogQueue = NewHTTPAccessLogQueue()
|
||||
|
||||
// HTTP访问日志队列
|
||||
// HTTPAccessLogQueue HTTP访问日志队列
|
||||
type HTTPAccessLogQueue struct {
|
||||
queue chan *pb.HTTPAccessLog
|
||||
}
|
||||
|
||||
// 获取新对象
|
||||
// NewHTTPAccessLogQueue 获取新对象
|
||||
func NewHTTPAccessLogQueue() *HTTPAccessLogQueue {
|
||||
// 队列中最大的值,超出此数量的访问日志会被抛弃
|
||||
// 队列中最大的值,超出此数量的访问日志会被丢弃
|
||||
// TODO 需要可以在界面中设置
|
||||
maxSize := 10000
|
||||
queue := &HTTPAccessLogQueue{
|
||||
@@ -27,7 +27,7 @@ func NewHTTPAccessLogQueue() *HTTPAccessLogQueue {
|
||||
return queue
|
||||
}
|
||||
|
||||
// 开始处理访问日志
|
||||
// Start 开始处理访问日志
|
||||
func (this *HTTPAccessLogQueue) Start() {
|
||||
ticker := time.NewTicker(1 * time.Second)
|
||||
for range ticker.C {
|
||||
@@ -38,7 +38,7 @@ func (this *HTTPAccessLogQueue) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
// 加入新访问日志
|
||||
// Push 加入新访问日志
|
||||
func (this *HTTPAccessLogQueue) Push(accessLog *pb.HTTPAccessLog) {
|
||||
select {
|
||||
case this.queue <- accessLog:
|
||||
@@ -77,7 +77,7 @@ Loop:
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{AccessLogs: accessLogs})
|
||||
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,10 +6,12 @@ import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/pires/go-proxyproto"
|
||||
"net"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -37,7 +39,7 @@ func NewHTTPClientPool() *HTTPClientPool {
|
||||
}
|
||||
|
||||
// Client 根据地址获取客户端
|
||||
func (this *HTTPClientPool) Client(req *http.Request, origin *serverconfigs.OriginConfig, originAddr string) (rawClient *http.Client, err error) {
|
||||
func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.OriginConfig, originAddr string, proxyProtocol *serverconfigs.ProxyProtocolConfig) (rawClient *http.Client, err error) {
|
||||
if origin.Addr == nil {
|
||||
return nil, errors.New("origin addr should not be empty (originId:" + strconv.FormatInt(origin.Id, 10) + ")")
|
||||
}
|
||||
@@ -73,11 +75,11 @@ func (this *HTTPClientPool) Client(req *http.Request, origin *serverconfigs.Orig
|
||||
numberCPU = 8
|
||||
}
|
||||
if maxConnections <= 0 {
|
||||
maxConnections = numberCPU * 8
|
||||
maxConnections = numberCPU * 32
|
||||
}
|
||||
|
||||
if idleConns <= 0 {
|
||||
idleConns = numberCPU * 4
|
||||
idleConns = numberCPU * 8
|
||||
}
|
||||
//logs.Println("[ORIGIN]max connections:", maxConnections)
|
||||
|
||||
@@ -105,7 +107,7 @@ func (this *HTTPClientPool) Client(req *http.Request, origin *serverconfigs.Orig
|
||||
for i := 1; i <= retries; i++ {
|
||||
port := int(toaConfig.RandLocalPort())
|
||||
// TODO 思考是否支持X-Real-IP/X-Forwarded-IP
|
||||
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + req.RemoteAddr)
|
||||
err := sharedTOAManager.SendMsg("add:" + strconv.Itoa(port) + ":" + req.requestRemoteAddr(true))
|
||||
if err != nil {
|
||||
remotelogs.Error("TOA", "add failed: "+err.Error())
|
||||
} else {
|
||||
@@ -126,10 +128,43 @@ func (this *HTTPClientPool) Client(req *http.Request, origin *serverconfigs.Orig
|
||||
}
|
||||
|
||||
// 普通的连接
|
||||
return (&net.Dialer{
|
||||
conn, err := (&net.Dialer{
|
||||
Timeout: connectionTimeout,
|
||||
KeepAlive: 1 * time.Minute,
|
||||
}).DialContext(ctx, network, originAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if proxyProtocol != nil && proxyProtocol.IsOn && (proxyProtocol.Version == serverconfigs.ProxyProtocolVersion1 || proxyProtocol.Version == serverconfigs.ProxyProtocolVersion2) {
|
||||
var remoteAddr = req.requestRemoteAddr(true)
|
||||
var transportProtocol = proxyproto.TCPv4
|
||||
if strings.Contains(remoteAddr, ":") {
|
||||
transportProtocol = proxyproto.TCPv6
|
||||
}
|
||||
var destAddr = conn.RemoteAddr()
|
||||
var reqConn = req.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if reqConn != nil {
|
||||
destAddr = reqConn.(net.Conn).LocalAddr()
|
||||
}
|
||||
header := proxyproto.Header{
|
||||
Version: byte(proxyProtocol.Version),
|
||||
Command: proxyproto.PROXY,
|
||||
TransportProtocol: transportProtocol,
|
||||
SourceAddr: &net.TCPAddr{
|
||||
IP: net.ParseIP(remoteAddr),
|
||||
Port: req.requestRemotePort(),
|
||||
},
|
||||
DestinationAddr: destAddr,
|
||||
}
|
||||
_, err = header.WriteTo(conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
},
|
||||
MaxIdleConns: 0,
|
||||
MaxIdleConnsPerHost: idleConns,
|
||||
|
||||
@@ -21,14 +21,14 @@ func TestHTTPClientPool_Client(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
{
|
||||
client, err := pool.Client(nil, origin, origin.Addr.PickAddress())
|
||||
client, err := pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("client:", client)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
client, err := pool.Client(nil, origin, origin.Addr.PickAddress())
|
||||
client, err := pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
t.Log("get", i)
|
||||
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress())
|
||||
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,6 @@ func BenchmarkHTTPClientPool_Client(b *testing.B) {
|
||||
|
||||
pool := NewHTTPClientPool()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress())
|
||||
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,11 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/metrics"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"golang.org/x/net/http2"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -30,6 +31,9 @@ var bytePool1k = utils.NewBytePool(20480, 1024)
|
||||
var bytePool32k = utils.NewBytePool(20480, 32*1024)
|
||||
var bytePool128k = utils.NewBytePool(20480, 128*1024)
|
||||
|
||||
// errors
|
||||
var errWritingToClient = errors.New("writing to client error")
|
||||
|
||||
// HTTPRequest HTTP请求
|
||||
type HTTPRequest struct {
|
||||
// 外部参数
|
||||
@@ -43,6 +47,7 @@ type HTTPRequest struct {
|
||||
IsHTTPS bool
|
||||
|
||||
// 内部参数
|
||||
isSubRequest bool
|
||||
writer *HTTPWriter
|
||||
web *serverconfigs.HTTPWebConfig // Web配置,重要提示:由于引用了别的共享的配置,所以操作中只能读取不要修改
|
||||
reverseProxyRef *serverconfigs.ReverseProxyRef // 反向代理引用
|
||||
@@ -61,12 +66,17 @@ type HTTPRequest struct {
|
||||
rewriteIsExternalURL bool // 重写目标是否为外部URL
|
||||
cacheRef *serverconfigs.HTTPCacheRef // 缓存设置
|
||||
cacheKey string // 缓存使用的Key
|
||||
isCached bool // 是否已经被缓存
|
||||
isAttack bool // 是否是攻击请求
|
||||
bodyData []byte // 读取的Body内容
|
||||
|
||||
// WAF相关
|
||||
firewallPolicyId int64
|
||||
firewallRuleGroupId int64
|
||||
firewallRuleSetId int64
|
||||
firewallRuleId int64
|
||||
firewallActions []string
|
||||
tags []string
|
||||
|
||||
logAttrs map[string]string
|
||||
|
||||
@@ -111,7 +121,25 @@ func (this *HTTPRequest) Do() {
|
||||
// Web配置
|
||||
err := this.configureWeb(this.Server.Web, true, 0)
|
||||
if err != nil {
|
||||
this.write500(err)
|
||||
this.write50x(err, http.StatusInternalServerError)
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
|
||||
// 特殊URL处理
|
||||
if len(this.rawURI) > 1 && this.rawURI[1] == '.' {
|
||||
// ACME
|
||||
// TODO 需要配置是否启用ACME检测
|
||||
if strings.HasPrefix(this.rawURI, "/.well-known/acme-challenge/") {
|
||||
this.doACME()
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 带宽限制
|
||||
if this.Server.BandwidthLimit != nil && this.Server.BandwidthLimit.IsOn && !this.Server.BandwidthLimit.IsEmpty() && this.Server.BandwidthLimitStatus != nil && this.Server.BandwidthLimitStatus.IsValid() {
|
||||
this.doBandwidthLimit()
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
@@ -125,18 +153,24 @@ func (this *HTTPRequest) Do() {
|
||||
}
|
||||
|
||||
// 访问控制
|
||||
// TODO 需要实现
|
||||
if !this.isSubRequest && this.web.Auth != nil && this.web.Auth.IsOn {
|
||||
if this.doAuth() {
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 自动跳转到HTTPS
|
||||
if this.IsHTTP && this.web.RedirectToHttps != nil && this.web.RedirectToHttps.IsOn {
|
||||
this.doRedirectToHTTPS(this.web.RedirectToHttps)
|
||||
this.doEnd()
|
||||
return
|
||||
if this.doRedirectToHTTPS(this.web.RedirectToHttps) {
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Gzip
|
||||
if this.web.GzipRef != nil && this.web.GzipRef.IsOn && this.web.Gzip != nil && this.web.Gzip.IsOn && this.web.Gzip.Level > 0 {
|
||||
this.writer.Gzip(this.web.Gzip)
|
||||
// Compression
|
||||
if this.web.Compression != nil && this.web.Compression.IsOn && this.web.Compression.Level > 0 {
|
||||
this.writer.SetCompression(this.web.Compression)
|
||||
}
|
||||
|
||||
// 开始调用
|
||||
@@ -151,6 +185,14 @@ func (this *HTTPRequest) Do() {
|
||||
|
||||
// 开始调用
|
||||
func (this *HTTPRequest) doBegin() {
|
||||
// 处理健康检查
|
||||
var healthCheckKey = this.RawReq.Header.Get(serverconfigs.HealthCheckHeaderName)
|
||||
if len(healthCheckKey) > 0 {
|
||||
if this.doHealthCheck(healthCheckKey) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 统计
|
||||
if this.web.StatRef != nil && this.web.StatRef.IsOn {
|
||||
this.doStat()
|
||||
@@ -163,16 +205,6 @@ func (this *HTTPRequest) doBegin() {
|
||||
}
|
||||
}
|
||||
|
||||
// 特殊URL处理
|
||||
if len(this.rawURI) > 1 && this.rawURI[1] == '.' {
|
||||
// ACME
|
||||
// TODO 需要配置是否启用ACME检测
|
||||
if strings.HasPrefix(this.rawURI, "/.well-known/acme-challenge/") {
|
||||
this.doACME()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 临时关闭页面
|
||||
if this.web.Shutdown != nil && this.web.Shutdown.IsOn {
|
||||
this.doShutdown()
|
||||
@@ -231,7 +263,20 @@ func (this *HTTPRequest) doEnd() {
|
||||
// 流量统计
|
||||
// TODO 增加是否开启开关
|
||||
if this.Server != nil {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.writer.sentBodyBytes)
|
||||
if this.isCached {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, this.writer.sentBodyBytes, 1, 1, 0, 0)
|
||||
} else {
|
||||
if this.isAttack {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 1, this.writer.sentBodyBytes)
|
||||
} else {
|
||||
stats.SharedTrafficStatManager.Add(this.Server.Id, this.Host, this.writer.sentBodyBytes, 0, 1, 0, 0, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 指标
|
||||
if metrics.SharedManager.HasHTTPMetrics() {
|
||||
this.doMetricsResponse()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -293,6 +338,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
|
||||
this.web.Root = web.Root
|
||||
}
|
||||
|
||||
// remote addr
|
||||
if web.RemoteAddr != nil && (web.RemoteAddr.IsPrior || isTop) && web.RemoteAddr.IsOn {
|
||||
this.web.RemoteAddr = web.RemoteAddr
|
||||
}
|
||||
|
||||
// charset
|
||||
if web.Charset != nil && (web.Charset.IsPrior || isTop) {
|
||||
this.web.Charset = web.Charset
|
||||
@@ -304,10 +354,14 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
|
||||
this.web.Websocket = web.Websocket
|
||||
}
|
||||
|
||||
// gzip
|
||||
if web.GzipRef != nil && (web.GzipRef.IsPrior || isTop) {
|
||||
this.web.GzipRef = web.GzipRef
|
||||
this.web.Gzip = web.Gzip
|
||||
// compression
|
||||
if web.Compression != nil && (web.Compression.IsPrior || isTop) {
|
||||
this.web.Compression = web.Compression
|
||||
}
|
||||
|
||||
// webp
|
||||
if web.WebP != nil && (web.WebP.IsPrior || isTop) {
|
||||
this.web.WebP = web.WebP
|
||||
}
|
||||
|
||||
// cache
|
||||
@@ -344,6 +398,11 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
|
||||
this.web.FastcgiList = web.FastcgiList
|
||||
}
|
||||
|
||||
// auth
|
||||
if web.Auth != nil && (web.Auth.IsPrior || isTop) {
|
||||
this.web.Auth = web.Auth
|
||||
}
|
||||
|
||||
// 重写规则
|
||||
if len(web.RewriteRefs) > 0 {
|
||||
for index, ref := range web.RewriteRefs {
|
||||
@@ -467,7 +526,9 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
case "edgeVersion":
|
||||
return teaconst.Version
|
||||
case "remoteAddr":
|
||||
return this.requestRemoteAddr()
|
||||
return this.requestRemoteAddr(true)
|
||||
case "remoteAddrValue":
|
||||
return this.requestRemoteAddr(false)
|
||||
case "rawRemoteAddr":
|
||||
addr := this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
@@ -481,9 +542,15 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
return this.requestRemoteUser()
|
||||
case "requestURI", "requestUri":
|
||||
return this.rawURI
|
||||
case "requestURL":
|
||||
var scheme = "http"
|
||||
if this.IsHTTPS {
|
||||
scheme = "https"
|
||||
}
|
||||
return scheme + "://" + this.Host + this.rawURI
|
||||
case "requestPath":
|
||||
return this.requestPath()
|
||||
case "requestPathExtension": // TODO 需要添加到文档中
|
||||
case "requestPathExtension":
|
||||
return filepath.Ext(this.requestPath())
|
||||
case "requestLength":
|
||||
return strconv.FormatInt(this.requestLength(), 10)
|
||||
@@ -530,6 +597,12 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
return this.Host
|
||||
case "referer":
|
||||
return this.RawReq.Referer()
|
||||
case "referer.host":
|
||||
u, err := url.Parse(this.RawReq.Referer())
|
||||
if err == nil {
|
||||
return u.Host
|
||||
}
|
||||
return ""
|
||||
case "userAgent":
|
||||
return this.RawReq.UserAgent()
|
||||
case "contentType":
|
||||
@@ -578,7 +651,6 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
}
|
||||
|
||||
// response.
|
||||
// TODO 需要在文档中添加说明
|
||||
if prefix == "response" {
|
||||
switch suffix {
|
||||
case "contentType":
|
||||
@@ -712,22 +784,36 @@ func (this *HTTPRequest) addVarMapping(varMapping map[string]string) {
|
||||
}
|
||||
|
||||
// 获取请求的客户端地址
|
||||
func (this *HTTPRequest) requestRemoteAddr() string {
|
||||
func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
if supportVar &&
|
||||
this.web.RemoteAddr != nil &&
|
||||
this.web.RemoteAddr.IsOn &&
|
||||
!this.web.RemoteAddr.IsEmpty() {
|
||||
var remoteAddr = this.Format(this.web.RemoteAddr.Value)
|
||||
if net.ParseIP(remoteAddr) != nil {
|
||||
return remoteAddr
|
||||
}
|
||||
}
|
||||
|
||||
// X-Forwarded-For
|
||||
forwardedFor := this.RawReq.Header.Get("X-Forwarded-For")
|
||||
if len(forwardedFor) > 0 {
|
||||
commaIndex := strings.Index(forwardedFor, ",")
|
||||
if commaIndex > 0 {
|
||||
return forwardedFor[:commaIndex]
|
||||
forwardedFor = forwardedFor[:commaIndex]
|
||||
}
|
||||
if net.ParseIP(forwardedFor) != nil {
|
||||
return forwardedFor
|
||||
}
|
||||
return forwardedFor
|
||||
}
|
||||
|
||||
// Real-IP
|
||||
{
|
||||
realIP, ok := this.RawReq.Header["X-Real-IP"]
|
||||
if ok && len(realIP) > 0 {
|
||||
return realIP[0]
|
||||
if net.ParseIP(realIP[0]) != nil {
|
||||
return realIP[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -735,7 +821,9 @@ func (this *HTTPRequest) requestRemoteAddr() string {
|
||||
{
|
||||
realIP, ok := this.RawReq.Header["X-Real-Ip"]
|
||||
if ok && len(realIP) > 0 {
|
||||
return realIP[0]
|
||||
if net.ParseIP(realIP[0]) != nil {
|
||||
return realIP[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -912,6 +1000,11 @@ func (this *HTTPRequest) requestServerPort() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// 获取完整的URL
|
||||
func (this *HTTPRequest) requestFullURL() string {
|
||||
return this.requestScheme() + "://" + this.Host + this.uri
|
||||
}
|
||||
|
||||
// 设置代理相关头部信息
|
||||
// 参考:https://tools.ietf.org/html/rfc7239
|
||||
func (this *HTTPRequest) setForwardHeaders(header http.Header) {
|
||||
@@ -1142,17 +1235,31 @@ func (this *HTTPRequest) canIgnore(err error) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// 已读到头
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
return true
|
||||
}
|
||||
|
||||
// 网络错误
|
||||
_, ok := err.(*net.OpError)
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
|
||||
// 客户端主动取消
|
||||
if err == context.Canceled {
|
||||
if err == errWritingToClient || err == context.Canceled || err == io.ErrShortWrite || strings.Contains(err.Error(), "write: connection timed out") || strings.Contains(err.Error(), "write: broken pipe") {
|
||||
return true
|
||||
}
|
||||
|
||||
// HTTP/2流错误
|
||||
{
|
||||
_, ok := err.(http2.StreamError)
|
||||
if ok {
|
||||
return true
|
||||
}
|
||||
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
|
||||
return true
|
||||
}
|
||||
|
||||
// HTTP内部错误
|
||||
if strings.HasPrefix(err.Error(), "http:") || strings.HasPrefix(err.Error(), "http2:") {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -12,6 +12,10 @@ func (this *HTTPRequest) doACME() {
|
||||
// TODO 对请求进行校验,防止恶意攻击
|
||||
|
||||
token := filepath.Base(this.RawReq.URL.Path)
|
||||
if token == "acme-challenge" || len(token) <= 32 {
|
||||
this.writer.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
|
||||
61
internal/nodes/http_request_auth.go
Normal file
61
internal/nodes/http_request_auth.go
Normal file
@@ -0,0 +1,61 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// 执行认证
|
||||
func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
if this.web.Auth == nil || !this.web.Auth.IsOn {
|
||||
return
|
||||
}
|
||||
|
||||
for _, ref := range this.web.Auth.PolicyRefs {
|
||||
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
|
||||
continue
|
||||
}
|
||||
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
|
||||
subReq.TLS = this.RawReq.TLS
|
||||
subReq.RemoteAddr = this.RawReq.RemoteAddr
|
||||
subReq.Host = this.RawReq.Host
|
||||
subReq.Proto = this.RawReq.Proto
|
||||
subReq.ProtoMinor = this.RawReq.ProtoMinor
|
||||
subReq.ProtoMajor = this.RawReq.ProtoMajor
|
||||
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
|
||||
subReq.Header.Set("Referer", this.requestFullURL())
|
||||
var writer = NewEmptyResponseWriter(this.writer)
|
||||
this.doSubRequest(writer, subReq)
|
||||
return writer.StatusCode(), nil
|
||||
}, this.Format)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if b {
|
||||
return
|
||||
} else {
|
||||
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
|
||||
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.Host
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
this.writer.WriteHeader(http.StatusUnauthorized)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
24
internal/nodes/http_request_bandwidth_limit.go
Normal file
24
internal/nodes/http_request_bandwidth_limit.go
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
)
|
||||
|
||||
// 带宽限制
|
||||
func (this *HTTPRequest) doBandwidthLimit() {
|
||||
var config = this.Server.BandwidthLimit
|
||||
|
||||
this.tags = append(this.tags, "bandwidth")
|
||||
|
||||
var statusCode = 509
|
||||
this.processResponseHeaders(statusCode)
|
||||
|
||||
this.writer.WriteHeader(statusCode)
|
||||
if len(config.NoticePageBody) != 0 {
|
||||
_, _ = this.writer.WriteString(config.NoticePageBody)
|
||||
} else {
|
||||
_, _ = this.writer.WriteString(serverconfigs.DefaultBandwidthLimitNoticePageBody)
|
||||
}
|
||||
}
|
||||
@@ -3,16 +3,20 @@ package nodes
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 读取缓存
|
||||
func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
cachePolicy := sharedNodeConfig.HTTPCachePolicy
|
||||
cachePolicy := this.Server.HTTPCachePolicy
|
||||
if cachePolicy == nil || !cachePolicy.IsOn {
|
||||
return
|
||||
}
|
||||
@@ -20,6 +24,12 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
if this.web.Cache == nil || !this.web.Cache.IsOn || (len(cachePolicy.CacheRefs) == 0 && len(this.web.Cache.CacheRefs) == 0) {
|
||||
return
|
||||
}
|
||||
|
||||
// 判断是否在预热
|
||||
if (strings.HasPrefix(this.RawReq.RemoteAddr, "127.") || strings.HasPrefix(this.RawReq.RemoteAddr, "[::1]")) && this.RawReq.Header.Get("X-Cache-Action") == "preheat" {
|
||||
return
|
||||
}
|
||||
|
||||
var addStatusHeader = this.web.Cache.AddStatusHeader
|
||||
if addStatusHeader {
|
||||
defer func() {
|
||||
@@ -39,6 +49,9 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
continue
|
||||
}
|
||||
if cacheRef.Conds.MatchRequest(this.Format) {
|
||||
if cacheRef.IsReverse {
|
||||
return
|
||||
}
|
||||
this.cacheRef = cacheRef
|
||||
refType = "server"
|
||||
break
|
||||
@@ -53,6 +66,9 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
continue
|
||||
}
|
||||
if cacheRef.Conds.MatchRequest(this.Format) {
|
||||
if cacheRef.IsReverse {
|
||||
return
|
||||
}
|
||||
this.cacheRef = cacheRef
|
||||
refType = "policy"
|
||||
break
|
||||
@@ -85,6 +101,7 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
this.cacheRef = nil
|
||||
return
|
||||
}
|
||||
|
||||
this.cacheKey = key
|
||||
|
||||
// 读取缓存
|
||||
@@ -94,23 +111,63 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
return
|
||||
}
|
||||
|
||||
// 判断是否在Purge
|
||||
if this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey {
|
||||
err := storage.Delete(key)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
|
||||
}
|
||||
|
||||
go func() {
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err == nil {
|
||||
for _, rpcServerService := range rpcClient.ServerRPCList() {
|
||||
_, err = rpcServerService.PurgeServerCache(rpcClient.Context(), &pb.PurgeServerCacheRequest{
|
||||
Domains: []string{this.Host},
|
||||
Keys: []string{key},
|
||||
Prefixes: nil,
|
||||
})
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
buf := bytePool32k.Get()
|
||||
defer func() {
|
||||
bytePool32k.Put(buf)
|
||||
}()
|
||||
|
||||
reader, err := storage.OpenReader(key)
|
||||
if err != nil {
|
||||
if err == caches.ErrNotFound {
|
||||
// cache相关变量
|
||||
this.varMapping["cache.status"] = "MISS"
|
||||
var reader caches.Reader
|
||||
var err error
|
||||
|
||||
// 是否优先检查WebP
|
||||
if this.web.WebP != nil &&
|
||||
this.web.WebP.IsOn &&
|
||||
this.web.WebP.MatchRequest(filepath.Ext(this.requestPath()), this.Format) &&
|
||||
this.web.WebP.MatchAccept(this.requestHeader("Accept")) {
|
||||
reader, _ = storage.OpenReader(key + webpSuffix)
|
||||
}
|
||||
|
||||
// 检查正常的文件
|
||||
if reader == nil {
|
||||
reader, err = storage.OpenReader(key)
|
||||
if err != nil {
|
||||
if err == caches.ErrNotFound {
|
||||
// cache相关变量
|
||||
this.varMapping["cache.status"] = "MISS"
|
||||
return
|
||||
}
|
||||
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = reader.Close()
|
||||
@@ -142,7 +199,7 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
})
|
||||
if err != nil {
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -150,6 +207,48 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
if addStatusHeader {
|
||||
this.writer.Header().Set("X-Cache", "HIT, "+refType+", "+reader.TypeName())
|
||||
}
|
||||
|
||||
// ETag
|
||||
// 这里强制设置ETag,如果先前源站设置了ETag,将会被覆盖,避免因为源站的ETag导致源站返回304 Not Modified
|
||||
var respHeader = this.writer.Header()
|
||||
var eTag = ""
|
||||
var lastModifiedAt = reader.LastModified()
|
||||
if lastModifiedAt > 0 {
|
||||
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
|
||||
respHeader.Del("Etag")
|
||||
respHeader["ETag"] = []string{eTag}
|
||||
}
|
||||
|
||||
// 支持 Last-Modified
|
||||
// 这里强制设置Last-Modified,如果先前源站设置了Last-Modified,将会被覆盖,避免因为源站的Last-Modified导致源站返回304 Not Modified
|
||||
var modifiedTime = ""
|
||||
if lastModifiedAt > 0 {
|
||||
modifiedTime = time.Unix(lastModifiedAt, 0).Format("Mon, 02 Jan 2006 15:04:05 GMT")
|
||||
respHeader.Set("Last-Modified", modifiedTime)
|
||||
}
|
||||
|
||||
// 支持 If-None-Match
|
||||
if len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
|
||||
// 自定义Header
|
||||
this.processResponseHeaders(http.StatusNotModified)
|
||||
this.writer.WriteHeader(http.StatusNotModified)
|
||||
this.isCached = true
|
||||
this.cacheRef = nil
|
||||
this.writer.SetOk()
|
||||
return true
|
||||
}
|
||||
|
||||
// 支持 If-Modified-Since
|
||||
if len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
|
||||
// 自定义Header
|
||||
this.processResponseHeaders(http.StatusNotModified)
|
||||
this.writer.WriteHeader(http.StatusNotModified)
|
||||
this.isCached = true
|
||||
this.cacheRef = nil
|
||||
this.writer.SetOk()
|
||||
return true
|
||||
}
|
||||
|
||||
this.processResponseHeaders(reader.Status())
|
||||
|
||||
// 输出Body
|
||||
@@ -223,7 +322,7 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
err = reader.ReadBodyRange(buf, rangeSet[0][0], rangeSet[0][1], func(n int) (goNext bool, err error) {
|
||||
_, err = this.writer.Write(buf[:n])
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, errWritingToClient
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
@@ -234,7 +333,7 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
return true
|
||||
}
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -253,31 +352,34 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
_, err = this.writer.WriteString("\r\n--" + boundary + "\r\n")
|
||||
}
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
// 不提示写入客户端错误
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(reader.BodySize(), 10) + "\r\n")
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
// 不提示写入客户端错误
|
||||
return true
|
||||
}
|
||||
|
||||
if len(contentType) > 0 {
|
||||
_, err = this.writer.WriteString("Content-Type: " + contentType + "\r\n\r\n")
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
// 不提示写入客户端错误
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
err := reader.ReadBodyRange(buf, set[0], set[1], func(n int) (goNext bool, err error) {
|
||||
_, err = this.writer.Write(buf[:n])
|
||||
return true, err
|
||||
if err != nil {
|
||||
return false, errWritingToClient
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -285,28 +387,33 @@ func (this *HTTPRequest) doCacheRead() (shouldStop bool) {
|
||||
|
||||
_, err = this.writer.WriteString("\r\n--" + boundary + "--\r\n")
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
// 不提示写入客户端错误
|
||||
return true
|
||||
}
|
||||
} else { // 没有Range
|
||||
this.writer.PrepareCompression(reader.BodySize())
|
||||
this.writer.WriteHeader(reader.Status())
|
||||
|
||||
err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
|
||||
_, err = this.writer.Write(buf[:n])
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, errWritingToClient
|
||||
}
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
if !this.canIgnore(err) {
|
||||
remotelogs.Error("REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_CACHE", "read from cache failed: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.cacheRef = nil // 终止读取不再往下传递
|
||||
this.isCached = true
|
||||
this.cacheRef = nil
|
||||
|
||||
this.writer.SetOk()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
@@ -17,30 +18,15 @@ func (this *HTTPRequest) write404() {
|
||||
_, _ = this.writer.Write([]byte(msg))
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) write500(err error) {
|
||||
func (this *HTTPRequest) write50x(err error, statusCode int) {
|
||||
if err != nil {
|
||||
this.addError(err)
|
||||
}
|
||||
|
||||
statusCode := http.StatusInternalServerError
|
||||
if this.doPage(statusCode) {
|
||||
return
|
||||
}
|
||||
this.processResponseHeaders(statusCode)
|
||||
this.writer.WriteHeader(statusCode)
|
||||
_, _ = this.writer.Write([]byte(http.StatusText(statusCode)))
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) write502(err error) {
|
||||
if err != nil {
|
||||
this.addError(err)
|
||||
}
|
||||
|
||||
statusCode := http.StatusBadGateway
|
||||
if this.doPage(statusCode) {
|
||||
return
|
||||
}
|
||||
this.processResponseHeaders(statusCode)
|
||||
this.writer.WriteHeader(statusCode)
|
||||
_, _ = this.writer.Write([]byte("502 Bad Gateway"))
|
||||
_, _ = this.writer.Write([]byte(types.String(statusCode) + " " + http.StatusText(statusCode)))
|
||||
}
|
||||
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/iwind/gofcgi/pkg"
|
||||
"github.com/iwind/gofcgi/pkg/fcgi"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -40,7 +41,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
}
|
||||
|
||||
if !env.Has("REMOTE_ADDR") {
|
||||
env["REMOTE_ADDR"] = this.requestRemoteAddr()
|
||||
env["REMOTE_ADDR"] = this.requestRemoteAddr(true)
|
||||
}
|
||||
if !env.Has("QUERY_STRING") {
|
||||
u, err := url.ParseRequestURI(this.uri)
|
||||
@@ -78,9 +79,9 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
poolSize = 32
|
||||
}
|
||||
|
||||
client, err := pkg.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
|
||||
client, err := fcgi.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
|
||||
if err != nil {
|
||||
this.write500(err)
|
||||
this.write50x(err, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -151,20 +152,20 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
params["HTTP_HOST"] = this.Host
|
||||
}
|
||||
|
||||
fcgiReq := pkg.NewRequest()
|
||||
fcgiReq := fcgi.NewRequest()
|
||||
fcgiReq.SetTimeout(fastcgi.ReadTimeoutDuration())
|
||||
fcgiReq.SetParams(params)
|
||||
fcgiReq.SetBody(this.RawReq.Body, uint32(this.requestLength()))
|
||||
|
||||
resp, stderr, err := client.Call(fcgiReq)
|
||||
if err != nil {
|
||||
this.write500(err)
|
||||
this.write50x(err, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if len(stderr) > 0 {
|
||||
err := errors.New("Fastcgi Error: " + strings.TrimSpace(string(stderr)) + " script: " + maps.NewMap(params).GetString("SCRIPT_FILENAME"))
|
||||
this.write500(err)
|
||||
this.write50x(err, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -200,14 +201,20 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
|
||||
pool.Put(buf)
|
||||
|
||||
err1 := resp.Body.Close()
|
||||
if err1 != nil {
|
||||
remotelogs.Warn("REQUEST_FASTCGI", err1.Error())
|
||||
closeErr := resp.Body.Close()
|
||||
if closeErr != nil {
|
||||
remotelogs.Warn("HTTP_REQUEST_FASTCGI", closeErr.Error())
|
||||
}
|
||||
|
||||
if err != nil && err != io.EOF {
|
||||
remotelogs.Warn("REQUEST_FASTCGI", err.Error())
|
||||
remotelogs.Warn("HTTP_REQUEST_FASTCGI", err.Error())
|
||||
this.addError(err)
|
||||
}
|
||||
|
||||
// 是否成功结束
|
||||
if err == nil && closeErr == nil {
|
||||
this.writer.SetOk()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
27
internal/nodes/http_request_health_check.go
Normal file
27
internal/nodes/http_request_health_check.go
Normal file
@@ -0,0 +1,27 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeutils"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
)
|
||||
|
||||
// 健康检查
|
||||
func (this *HTTPRequest) doHealthCheck(key string) (stop bool) {
|
||||
this.tags = append(this.tags, "healthCheck")
|
||||
|
||||
this.RawReq.Header.Del(serverconfigs.HealthCheckHeaderName)
|
||||
|
||||
data, err := nodeutils.DecryptData(sharedNodeConfig.NodeId, sharedNodeConfig.Secret, key)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_HEALTH_CHECK", "decode key failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if data.GetBool("onlyBasicRequest") {
|
||||
return true
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -13,6 +13,9 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
|
||||
if !u.IsOn {
|
||||
continue
|
||||
}
|
||||
if !u.MatchRequest(this.Format) {
|
||||
continue
|
||||
}
|
||||
if u.MatchPrefix { // 匹配前缀
|
||||
if strings.HasPrefix(fullURL, u.BeforeURL) {
|
||||
afterURL := u.AfterURL
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user