Compare commits
73 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57cb00edf0 | ||
|
|
3fb39b479a | ||
|
|
4a1daff143 | ||
|
|
dd1dbd424e | ||
|
|
305cb4b46e | ||
|
|
ef90dce29b | ||
|
|
3cb69f4c71 | ||
|
|
af4cd05df2 | ||
|
|
64e0ae80b7 | ||
|
|
8bba228745 | ||
|
|
8cc06e6707 | ||
|
|
52fdee2eeb | ||
|
|
b5f52dd136 | ||
|
|
abda886de5 | ||
|
|
18f08525b9 | ||
|
|
b2a9a31fe5 | ||
|
|
f578114aeb | ||
|
|
bf4f47fc35 | ||
|
|
0be951742a | ||
|
|
59d3d6ae4b | ||
|
|
d061876f7e | ||
|
|
ddaec82415 | ||
|
|
8afd00f00d | ||
|
|
0b306f0a22 | ||
|
|
b36a36172b | ||
|
|
770278bbbc | ||
|
|
ca24818571 | ||
|
|
cd0af22655 | ||
|
|
96cb8d8af7 | ||
|
|
91face15bf | ||
|
|
6f52df63a5 | ||
|
|
509d81dc66 | ||
|
|
817f2a6f91 | ||
|
|
d74e10c7a8 | ||
|
|
1a1280b76e | ||
|
|
85d3b1169f | ||
|
|
1b32292e0c | ||
|
|
e6ba44fb2f | ||
|
|
9b6e022f46 | ||
|
|
4e9ab19fc0 | ||
|
|
4d634d8fa5 | ||
|
|
a538282e4f | ||
|
|
df31921954 | ||
|
|
299abb7b04 | ||
|
|
85ce63b4d3 | ||
|
|
9d9ae288bd | ||
|
|
c0a35eb5e7 | ||
|
|
d1d0ff062b | ||
|
|
396e8a22c4 | ||
|
|
4f040db1ef | ||
|
|
e3cf111344 | ||
|
|
28cb3c383d | ||
|
|
10665c0f37 | ||
|
|
4096f11909 | ||
|
|
5df209b6d5 | ||
|
|
db353fe025 | ||
|
|
30c3e143b8 | ||
|
|
15fe7b33a4 | ||
|
|
17e0666ba4 | ||
|
|
3f24bfaaf5 | ||
|
|
ceadcfece9 | ||
|
|
bc706237ef | ||
|
|
b8c5a78f2e | ||
|
|
9ad1c3a3c8 | ||
|
|
8cfde43f5d | ||
|
|
6f843b071a | ||
|
|
a72b025900 | ||
|
|
dfb66775d7 | ||
|
|
c4dca2df30 | ||
|
|
a3525bdaa4 | ||
|
|
09390bbb97 | ||
|
|
70ae4391d7 | ||
|
|
85931f55e1 |
@@ -52,7 +52,6 @@ function build() {
|
||||
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
|
||||
cp -R "$ROOT"/www "$DIST"/
|
||||
cp -R "$ROOT"/pages "$DIST"/
|
||||
cp -R "$ROOT"/resources "$DIST"/
|
||||
|
||||
# we support TOA on linux/amd64 only
|
||||
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]
|
||||
|
||||
Binary file not shown.
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/apps"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
@@ -19,7 +20,7 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
app := apps.NewAppCmd().
|
||||
var app = apps.NewAppCmd().
|
||||
Version(teaconst.Version).
|
||||
Product(teaconst.ProductName).
|
||||
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog]").
|
||||
@@ -67,24 +68,30 @@ func main() {
|
||||
fmt.Println("done")
|
||||
})
|
||||
app.On("pprof", func() {
|
||||
// TODO 自己指定端口
|
||||
addr := "127.0.0.1:6060"
|
||||
var flagSet = flag.NewFlagSet("pprof", flag.ExitOnError)
|
||||
var addr string
|
||||
flagSet.StringVar(&addr, "addr", "", "")
|
||||
_ = flagSet.Parse(os.Args[2:])
|
||||
|
||||
if len(addr) == 0 {
|
||||
addr = "127.0.0.1:6060"
|
||||
}
|
||||
logs.Println("starting with pprof '" + addr + "'...")
|
||||
|
||||
go func() {
|
||||
err := http.ListenAndServe(addr, nil)
|
||||
if err != nil {
|
||||
logs.Println("[error]" + err.Error())
|
||||
logs.Println("[ERROR]" + err.Error())
|
||||
}
|
||||
}()
|
||||
|
||||
node := nodes.NewNode()
|
||||
var node = nodes.NewNode()
|
||||
node.Start()
|
||||
})
|
||||
app.On("dbstat", func() {
|
||||
teaconst.EnableDBStat = true
|
||||
|
||||
node := nodes.NewNode()
|
||||
var node = nodes.NewNode()
|
||||
node.Start()
|
||||
})
|
||||
app.On("trackers", func() {
|
||||
@@ -154,7 +161,7 @@ func main() {
|
||||
app.On("ip.drop", func() {
|
||||
var args = os.Args[2:]
|
||||
if len(args) == 0 {
|
||||
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS]")
|
||||
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS] [--async]")
|
||||
return
|
||||
}
|
||||
var ip = args[0]
|
||||
@@ -168,6 +175,11 @@ func main() {
|
||||
if ok {
|
||||
timeoutSeconds = types.Int(timeout[0])
|
||||
}
|
||||
var async = false
|
||||
_, ok = options["async"]
|
||||
if ok {
|
||||
async = true
|
||||
}
|
||||
|
||||
fmt.Println("drop ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
@@ -176,6 +188,7 @@ func main() {
|
||||
Params: map[string]interface{}{
|
||||
"ip": ip,
|
||||
"timeoutSeconds": timeoutSeconds,
|
||||
"async": async,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -305,8 +318,23 @@ func main() {
|
||||
}
|
||||
}
|
||||
})
|
||||
app.On("bandwidth", func() {
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
return
|
||||
}
|
||||
var statsMap = maps.NewMap(reply.Params).Get("stats")
|
||||
statsJSON, err := json.MarshalIndent(statsMap, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
return
|
||||
}
|
||||
fmt.Println(string(statsJSON))
|
||||
})
|
||||
app.Run(func() {
|
||||
node := nodes.NewNode()
|
||||
var node = nodes.NewNode()
|
||||
node.Start()
|
||||
})
|
||||
}
|
||||
|
||||
15
go.mod
15
go.mod
@@ -4,6 +4,7 @@ go 1.18
|
||||
|
||||
replace (
|
||||
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
|
||||
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -16,10 +17,10 @@ require (
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6
|
||||
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
|
||||
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
|
||||
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5
|
||||
github.com/klauspost/compress v1.15.8
|
||||
github.com/mattn/go-sqlite3 v1.14.9
|
||||
github.com/mdlayher/netlink v1.4.2
|
||||
@@ -27,20 +28,22 @@ require (
|
||||
github.com/mssola/user_agent v0.5.3
|
||||
github.com/pires/go-proxyproto v0.6.1
|
||||
github.com/shirou/gopsutil/v3 v3.22.2
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
|
||||
golang.org/x/text v0.3.7
|
||||
google.golang.org/grpc v1.45.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
rogchap.com/v8go v0.7.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v0.4.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/chai2010/webp v1.1.0 // indirect
|
||||
github.com/chai2010/webp v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-sql-driver/mysql v1.5.0 // indirect
|
||||
github.com/google/go-cmp v0.5.7 // indirect
|
||||
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
|
||||
|
||||
84
go.sum
84
go.sum
@@ -1,17 +1,13 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
|
||||
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
@@ -20,8 +16,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
|
||||
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
|
||||
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
|
||||
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
|
||||
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
|
||||
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
@@ -37,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -48,10 +43,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
@@ -83,20 +76,20 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
|
||||
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
|
||||
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
|
||||
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55 h1:shQNx0flJFBwKsGE7Hs3bI2bDz+YF0zl/4qE8B2KRiY=
|
||||
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
|
||||
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
|
||||
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
|
||||
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5 h1:SgKhrqRhWVpu0ZKxQM8MGjdhy7hlH3Xax8snwsZWSic=
|
||||
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
|
||||
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
|
||||
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
|
||||
@@ -109,14 +102,10 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ=
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
|
||||
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
|
||||
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
|
||||
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -150,23 +139,11 @@ github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb h1:2dC7L10LmTqlyMV
|
||||
github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
|
||||
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
|
||||
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
|
||||
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -180,9 +157,7 @@ github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtS
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
|
||||
@@ -196,41 +171,30 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
|
||||
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
@@ -259,21 +223,15 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -300,11 +258,10 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
@@ -316,7 +273,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
|
||||
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
|
||||
@@ -330,7 +286,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
|
||||
@@ -339,7 +294,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
|
||||
@@ -358,25 +312,19 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
|
||||
honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
|
||||
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
|
||||
rogchap.com/v8go v0.7.0 h1:kgjbiO4zE5itA962ze6Hqmbs4HgZbGzmueCXsZtremg=
|
||||
rogchap.com/v8go v0.7.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs=
|
||||
|
||||
@@ -89,7 +89,10 @@ func (this *LogWriter) Write(message string) {
|
||||
}
|
||||
}
|
||||
|
||||
this.c <- message
|
||||
select {
|
||||
case this.c <- message:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (this *LogWriter) Close() {
|
||||
|
||||
@@ -96,13 +96,13 @@ func (this *FileList) Reset() error {
|
||||
}
|
||||
|
||||
func (this *FileList) Add(hash string, item *Item) error {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := db.Add(hash, item)
|
||||
err := db.AddAsync(hash, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
}
|
||||
|
||||
func (this *FileList) Exist(hash string) (bool, error) {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// 如果Hash列表里不存在,那么必然不存在
|
||||
if !db.hashMap.Exist(hash) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var item = this.memoryCache.Read(hash)
|
||||
if item != nil {
|
||||
return true, nil
|
||||
@@ -225,9 +230,9 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
|
||||
return err
|
||||
}
|
||||
if notFound {
|
||||
_, err = db.deleteHitByHashStmt.Exec(hash)
|
||||
err = db.DeleteHitAsync(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
return db.WrapError(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,7 +271,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
|
||||
_ = check
|
||||
|
||||
row := db.statStmt.QueryRow()
|
||||
var row = db.statStmt.QueryRow()
|
||||
if row.Err() != nil {
|
||||
return nil, row.Err()
|
||||
}
|
||||
@@ -291,13 +296,13 @@ func (this *FileList) Count() (int64, error) {
|
||||
|
||||
// IncreaseHit 增加点击量
|
||||
func (this *FileList) IncreaseHit(hash string) error {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return db.IncreaseHit(hash)
|
||||
return db.IncreaseHitAsync(hash)
|
||||
}
|
||||
|
||||
// OnAdd 添加事件
|
||||
@@ -326,17 +331,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
|
||||
return fnv.HashString(hash) % CountFileDB
|
||||
}
|
||||
|
||||
func (this *FileList) getDB(hash string) *FileListDB {
|
||||
func (this *FileList) GetDB(hash string) *FileListDB {
|
||||
return this.dbList[fnv.HashString(hash)%CountFileDB]
|
||||
}
|
||||
|
||||
func (this *FileList) remove(hash string) (notFound bool, err error) {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// HashMap中不存在,则确定不存在
|
||||
if !db.hashMap.Exist(hash) {
|
||||
return true, nil
|
||||
}
|
||||
defer db.hashMap.Delete(hash)
|
||||
|
||||
// 从缓存中删除
|
||||
this.memoryCache.Delete(hash)
|
||||
|
||||
@@ -357,16 +368,16 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = db.deleteByHashStmt.Exec(hash)
|
||||
err = db.DeleteAsync(hash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, db.WrapError(err)
|
||||
}
|
||||
|
||||
atomic.AddInt64(&this.total, -1)
|
||||
|
||||
_, err = db.deleteHitByHashStmt.Exec(hash)
|
||||
err = db.DeleteHitAsync(hash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return false, db.WrapError(err)
|
||||
}
|
||||
|
||||
if this.onRemove != nil {
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
@@ -16,9 +18,15 @@ import (
|
||||
)
|
||||
|
||||
type FileListDB struct {
|
||||
dbPath string
|
||||
|
||||
readDB *dbs.DB
|
||||
writeDB *dbs.DB
|
||||
|
||||
writeBatch *dbs.Batch
|
||||
|
||||
hashMap *FileListHashMap
|
||||
|
||||
itemsTableName string
|
||||
hitsTableName string
|
||||
|
||||
@@ -28,29 +36,48 @@ type FileListDB struct {
|
||||
isReady bool
|
||||
|
||||
// cacheItems
|
||||
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
|
||||
insertStmt *dbs.Stmt // 写入数据
|
||||
selectByHashStmt *dbs.Stmt // 使用hash查询数据
|
||||
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
|
||||
|
||||
insertStmt *dbs.Stmt // 写入数据
|
||||
insertSQL string
|
||||
|
||||
selectByHashStmt *dbs.Stmt // 使用hash查询数据
|
||||
|
||||
selectHashListStmt *dbs.Stmt
|
||||
|
||||
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
deleteByHashSQL string
|
||||
|
||||
statStmt *dbs.Stmt // 统计
|
||||
purgeStmt *dbs.Stmt // 清理
|
||||
deleteAllStmt *dbs.Stmt // 删除所有数据
|
||||
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
|
||||
|
||||
// hits
|
||||
insertHitStmt *dbs.Stmt // 写入数据
|
||||
increaseHitStmt *dbs.Stmt // 增加点击量
|
||||
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
lfuHitsStmt *dbs.Stmt // 读取老的数据
|
||||
insertHitSQL string // 写入数据
|
||||
increaseHitSQL string // 增加点击量
|
||||
deleteHitByHashSQL string // 根据hash删除数据
|
||||
lfuHitsStmt *dbs.Stmt // 读取老的数据
|
||||
}
|
||||
|
||||
func NewFileListDB() *FileListDB {
|
||||
return &FileListDB{}
|
||||
return &FileListDB{
|
||||
hashMap: NewFileListHashMap(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListDB) Open(dbPath string) error {
|
||||
this.dbPath = dbPath
|
||||
|
||||
// 动态调整Cache值
|
||||
var cacheSize = 32000
|
||||
var memoryGB = utils.SystemMemoryGB()
|
||||
if memoryGB >= 8 {
|
||||
cacheSize += 32000 * memoryGB / 8
|
||||
}
|
||||
|
||||
// write db
|
||||
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
|
||||
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize)+"&_secure_delete=FAST")
|
||||
if err != nil {
|
||||
return errors.New("open write database failed: " + err.Error())
|
||||
}
|
||||
@@ -65,13 +92,22 @@ func (this *FileListDB) Open(dbPath string) error {
|
||||
}**/
|
||||
|
||||
this.writeDB = dbs.NewDB(writeDB)
|
||||
this.writeBatch = dbs.NewBatch(writeDB, 4)
|
||||
this.writeBatch.OnFail(func(err error) {
|
||||
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error())
|
||||
})
|
||||
|
||||
goman.New(func() {
|
||||
this.writeBatch.Exec()
|
||||
})
|
||||
|
||||
if teaconst.EnableDBStat {
|
||||
this.writeBatch.EnableStat(true)
|
||||
this.writeDB.EnableStat(true)
|
||||
}
|
||||
|
||||
// read db
|
||||
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
|
||||
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize))
|
||||
if err != nil {
|
||||
return errors.New("open read database failed: " + err.Error())
|
||||
}
|
||||
@@ -115,7 +151,8 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -125,7 +162,10 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
|
||||
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
|
||||
|
||||
this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
|
||||
this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -147,17 +187,11 @@ func (this *FileListDB) Init() error {
|
||||
|
||||
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
|
||||
|
||||
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
|
||||
this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
|
||||
|
||||
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
|
||||
|
||||
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
|
||||
|
||||
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
|
||||
if err != nil {
|
||||
@@ -166,6 +200,14 @@ func (this *FileListDB) Init() error {
|
||||
|
||||
this.isReady = true
|
||||
|
||||
// 加载HashMap
|
||||
go func() {
|
||||
err := this.hashMap.Load(this)
|
||||
if err != nil {
|
||||
remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -177,17 +219,47 @@ func (this *FileListDB) Total() int64 {
|
||||
return this.total
|
||||
}
|
||||
|
||||
func (this *FileListDB) Add(hash string, item *Item) error {
|
||||
func (this *FileListDB) AddAsync(hash string, item *Item) error {
|
||||
this.hashMap.Add(hash)
|
||||
|
||||
if item.StaleAt == 0 {
|
||||
item.StaleAt = item.ExpiredAt
|
||||
}
|
||||
|
||||
this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
this.hashMap.Add(hash)
|
||||
|
||||
if item.StaleAt == 0 {
|
||||
item.StaleAt = item.ExpiredAt
|
||||
}
|
||||
|
||||
// 放入队列
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
|
||||
if err != nil {
|
||||
return this.WrapError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteAsync(hash string) error {
|
||||
this.hashMap.Delete(hash)
|
||||
|
||||
this.writeBatch.Add(this.deleteByHashSQL, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteSync(hash string) error {
|
||||
this.hashMap.Delete(hash)
|
||||
|
||||
_, err := this.deleteByHashStmt.Exec(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -246,10 +318,36 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHit(hash string) error {
|
||||
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
|
||||
rows, err := this.selectHashListStmt.Query(lastId)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var id int64
|
||||
var hash string
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&id, &hash)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return
|
||||
}
|
||||
maxId = id
|
||||
hashList = append(hashList, hash)
|
||||
}
|
||||
|
||||
_ = rows.Close()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHitAsync(hash string) error {
|
||||
var week = timeutil.Format("YW")
|
||||
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
|
||||
return err
|
||||
this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteHitAsync(hash string) error {
|
||||
this.writeBatch.Add(this.deleteHitByHashSQL, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
@@ -262,7 +360,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
for {
|
||||
result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
|
||||
if err != nil {
|
||||
return err
|
||||
return this.WrapError(err)
|
||||
}
|
||||
affectedRows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
@@ -281,9 +379,11 @@ func (this *FileListDB) CleanAll() error {
|
||||
|
||||
_, err := this.deleteAllStmt.Exec()
|
||||
if err != nil {
|
||||
return err
|
||||
return this.WrapError(err)
|
||||
}
|
||||
|
||||
this.hashMap.Clean()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -300,6 +400,9 @@ func (this *FileListDB) Close() error {
|
||||
if this.selectByHashStmt != nil {
|
||||
_ = this.selectByHashStmt.Close()
|
||||
}
|
||||
if this.selectHashListStmt != nil {
|
||||
_ = this.selectHashListStmt.Close()
|
||||
}
|
||||
if this.deleteByHashStmt != nil {
|
||||
_ = this.deleteByHashStmt.Close()
|
||||
}
|
||||
@@ -315,16 +418,6 @@ func (this *FileListDB) Close() error {
|
||||
if this.listOlderItemsStmt != nil {
|
||||
_ = this.listOlderItemsStmt.Close()
|
||||
}
|
||||
|
||||
if this.insertHitStmt != nil {
|
||||
_ = this.insertHitStmt.Close()
|
||||
}
|
||||
if this.increaseHitStmt != nil {
|
||||
_ = this.increaseHitStmt.Close()
|
||||
}
|
||||
if this.deleteHitByHashStmt != nil {
|
||||
_ = this.deleteHitByHashStmt.Close()
|
||||
}
|
||||
if this.lfuHitsStmt != nil {
|
||||
_ = this.lfuHitsStmt.Close()
|
||||
}
|
||||
@@ -345,12 +438,23 @@ func (this *FileListDB) Close() error {
|
||||
}
|
||||
}
|
||||
|
||||
if this.writeBatch != nil {
|
||||
this.writeBatch.Close()
|
||||
}
|
||||
|
||||
if len(errStrings) == 0 {
|
||||
return nil
|
||||
}
|
||||
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
|
||||
}
|
||||
|
||||
func (this *FileListDB) WrapError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return errors.New(err.Error() + "(file: " + this.dbPath + ")")
|
||||
}
|
||||
|
||||
// 初始化
|
||||
func (this *FileListDB) initTables(times int) error {
|
||||
{
|
||||
@@ -393,10 +497,10 @@ ON "` + this.itemsTableName + `" (
|
||||
if dropErr == nil {
|
||||
return this.initTables(times + 1)
|
||||
}
|
||||
return err
|
||||
return this.WrapError(err)
|
||||
}
|
||||
|
||||
return err
|
||||
return this.WrapError(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -421,10 +525,10 @@ ON "` + this.hitsTableName + `" (
|
||||
if dropErr == nil {
|
||||
return this.initTables(times + 1)
|
||||
}
|
||||
return err
|
||||
return this.WrapError(err)
|
||||
}
|
||||
|
||||
return err
|
||||
return this.WrapError(err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
114
internal/caches/list_file_hash_map.go
Normal file
114
internal/caches/list_file_hash_map.go
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"math/big"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// FileListHashMap 文件Hash列表
|
||||
type FileListHashMap struct {
|
||||
m map[uint64]zero.Zero
|
||||
|
||||
locker sync.RWMutex
|
||||
isAvailable bool
|
||||
isReady bool
|
||||
}
|
||||
|
||||
func NewFileListHashMap() *FileListHashMap {
|
||||
return &FileListHashMap{
|
||||
m: map[uint64]zero.Zero{},
|
||||
isAvailable: false,
|
||||
isReady: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
// 如果系统内存过小,我们不缓存
|
||||
if utils.SystemMemoryGB() < 3 {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.isAvailable = true
|
||||
|
||||
var lastId int64
|
||||
for {
|
||||
hashList, maxId, err := db.ListHashes(lastId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(hashList) == 0 {
|
||||
break
|
||||
}
|
||||
this.AddHashes(hashList)
|
||||
lastId = maxId
|
||||
}
|
||||
|
||||
this.isReady = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Add(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
this.m[this.bigInt(hash)] = zero.New()
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
for _, hash := range hashes {
|
||||
this.m[this.bigInt(hash)] = zero.New()
|
||||
}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Delete(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
delete(this.m, this.bigInt(hash))
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Exist(hash string) bool {
|
||||
if !this.isAvailable {
|
||||
return true
|
||||
}
|
||||
if !this.isReady {
|
||||
// 只有完全Ready时才能判断是否为false
|
||||
return true
|
||||
}
|
||||
this.locker.RLock()
|
||||
_, ok := this.m[this.bigInt(hash)]
|
||||
this.locker.RUnlock()
|
||||
return ok
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Clean() {
|
||||
this.locker.Lock()
|
||||
this.m = map[uint64]zero.Zero{}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) bigInt(hash string) uint64 {
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
return bigInt.Uint64()
|
||||
}
|
||||
91
internal/caches/list_file_hash_map_test.go
Normal file
91
internal/caches/list_file_hash_map_test.go
Normal file
@@ -0,0 +1,91 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"math/big"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileListHashMap_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m.Add(stringutil.Md5(types.String(i)))
|
||||
}
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Memory2(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = map[uint64]zero.Zero{}
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m[uint64(i)] = zero.New()
|
||||
}
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
|
||||
}
|
||||
|
||||
func TestFileListHashMap_BigInt(t *testing.T) {
|
||||
for _, s := range []string{"1", "2", "3", "123", "123456"} {
|
||||
var hash = stringutil.Md5(s)
|
||||
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Load(t *testing.T) {
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
err = m.Load(list.GetDB("abc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Add("abc")
|
||||
|
||||
for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
|
||||
t.Log(hash, "=>", m.Exist(hash))
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark_BigInt(b *testing.B) {
|
||||
var hash = stringutil.Md5("123456")
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
_ = bigInt.Uint64()
|
||||
}
|
||||
}
|
||||
@@ -69,8 +69,8 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
_ = list.Close()
|
||||
}()
|
||||
|
||||
before := time.Now()
|
||||
for i := 0; i < 100_000; i++ {
|
||||
var before = time.Now()
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
|
||||
_ = list.Add(stringutil.Md5(u), &caches.Item{
|
||||
Key: u,
|
||||
@@ -290,12 +290,12 @@ func TestFileList_Stat(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileList_Count(t *testing.T) {
|
||||
list := caches.NewFileList(Tea.Root + "/data")
|
||||
var list = caches.NewFileList(Tea.Root + "/data")
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := time.Now()
|
||||
var before = time.Now()
|
||||
count, err := list.Count()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -367,6 +367,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strconv"
|
||||
"sync"
|
||||
@@ -16,7 +15,7 @@ var SharedManager = NewManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventQuit, func() {
|
||||
logs.Println("CACHE", "quiting cache manager")
|
||||
remotelogs.Println("CACHE", "quiting cache manager")
|
||||
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ package caches
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
// PartialRanges 内容分区范围定义
|
||||
@@ -30,7 +30,7 @@ func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
|
||||
}
|
||||
|
||||
func NewPartialRangesFromFile(path string) (*PartialRanges, error) {
|
||||
data, err := ioutil.ReadFile(path)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -116,12 +116,12 @@ func (this *PartialRanges) WriteToFile(path string) error {
|
||||
if err != nil {
|
||||
return errors.New("convert to json failed: " + err.Error())
|
||||
}
|
||||
return ioutil.WriteFile(path, data, 0666)
|
||||
return os.WriteFile(path, data, 0666)
|
||||
}
|
||||
|
||||
// ReadFromFile 从文件中读取
|
||||
func (this *PartialRanges) ReadFromFile(path string) (*PartialRanges, error) {
|
||||
data, err := ioutil.ReadFile(path)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -65,12 +65,13 @@ var sharedWritingFileKeyLocker = sync.Mutex{}
|
||||
|
||||
var maxOpenFiles = NewMaxOpenFiles()
|
||||
|
||||
const maxOpenFilesSlowCost = 5000 * time.Microsecond // 0.5ms
|
||||
const maxOpenFilesSlowCost = 1000 * time.Microsecond // us
|
||||
const protectingLoadWhenDump = false
|
||||
|
||||
// FileStorage 文件缓存
|
||||
// 文件结构:
|
||||
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
|
||||
//
|
||||
// 文件结构:
|
||||
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
|
||||
type FileStorage struct {
|
||||
policy *serverconfigs.HTTPCachePolicy
|
||||
options *serverconfigs.HTTPFileCacheStorage // 二级缓存
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
@@ -152,7 +152,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
|
||||
"Last-Modified": []string{"Wed, 06 Jan 2021 10:03:29 GMT"},
|
||||
"Server": []string{"CDN-Server"},
|
||||
},
|
||||
Body: ioutil.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
|
||||
Body: io.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
|
||||
}
|
||||
|
||||
for k, v := range resp.Header {
|
||||
|
||||
@@ -95,7 +95,8 @@ func (this *MemoryStorage) Init() error {
|
||||
// 启动定时Flush memory to disk任务
|
||||
if this.parentStorage != nil {
|
||||
// TODO 应该根据磁盘性能决定线程数
|
||||
var threads = 1
|
||||
// TODO 线程数应该可以在缓存策略和节点中设定
|
||||
var threads = runtime.NumCPU()
|
||||
|
||||
for i := 0; i < threads; i++ {
|
||||
goman.New(func() {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -304,3 +305,30 @@ func TestMemoryStorage_Stop(t *testing.T) {
|
||||
|
||||
t.Log(len(m))
|
||||
}
|
||||
|
||||
func BenchmarkValuesMap(b *testing.B) {
|
||||
var m = map[uint64]*MemoryItem{}
|
||||
var count = 1_000_000
|
||||
for i := 0; i < count; i++ {
|
||||
m[uint64(i)] = &MemoryItem{
|
||||
ExpiresAt: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
b.Log(len(m))
|
||||
|
||||
var locker = sync.Mutex{}
|
||||
b.ResetTimer()
|
||||
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
locker.Lock()
|
||||
_, ok := m[uint64(rands.Int(0, 1_000_000))]
|
||||
_ = ok
|
||||
locker.Unlock()
|
||||
|
||||
locker.Lock()
|
||||
delete(m, uint64(rands.Int(2, 1000000)))
|
||||
locker.Unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ package caches_test
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -16,7 +15,7 @@ func TestPartialFileWriter_Write(t *testing.T) {
|
||||
_ = os.Remove(path)
|
||||
|
||||
var reader = func() {
|
||||
data, err := ioutil.ReadFile(path)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"io"
|
||||
)
|
||||
@@ -10,6 +11,10 @@ import (
|
||||
var sharedBrotliReaderPool *ReaderPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"io"
|
||||
)
|
||||
@@ -10,6 +11,10 @@ import (
|
||||
var sharedDeflateReaderPool *ReaderPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"io"
|
||||
)
|
||||
@@ -10,6 +11,10 @@ import (
|
||||
var sharedGzipReaderPool *ReaderPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"io"
|
||||
)
|
||||
@@ -10,6 +11,10 @@ import (
|
||||
var sharedZSTDReaderPool *ReaderPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -5,10 +5,46 @@ package compressions_test
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/compressions"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBrotliWriter_LargeFile(t *testing.T) {
|
||||
var data = []byte{}
|
||||
for i := 0; i < 1024*1024; i++ {
|
||||
data = append(data, stringutil.Rand(32)...)
|
||||
}
|
||||
t.Log(len(data)/1024/1024, "M")
|
||||
|
||||
var before = time.Now()
|
||||
defer func() {
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
}()
|
||||
|
||||
var buf = &bytes.Buffer{}
|
||||
writer, err := compressions.NewBrotliWriter(buf, 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var offset = 0
|
||||
var size = 4096
|
||||
for offset < len(data) {
|
||||
_, err = writer.Write(data[offset : offset+size])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
offset += size
|
||||
}
|
||||
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkBrotliWriter_Write(b *testing.B) {
|
||||
var data = []byte(strings.Repeat("A", 1024))
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/andybalholm/brotli"
|
||||
"io"
|
||||
@@ -11,6 +12,10 @@ import (
|
||||
var sharedBrotliWriterPool *WriterPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -4,6 +4,7 @@ package compressions
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"io"
|
||||
)
|
||||
@@ -11,6 +12,10 @@ import (
|
||||
var sharedDeflateWriterPool *WriterPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -4,6 +4,7 @@ package compressions
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"io"
|
||||
)
|
||||
@@ -11,6 +12,10 @@ import (
|
||||
var sharedGzipWriterPool *WriterPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"io"
|
||||
@@ -11,6 +12,10 @@ import (
|
||||
var sharedZSTDWriterPool *WriterPool
|
||||
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
var maxSize = utils.SystemMemoryGB() * 256
|
||||
if maxSize == 0 {
|
||||
maxSize = 256
|
||||
|
||||
@@ -3,7 +3,7 @@ package configs
|
||||
import (
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"gopkg.in/yaml.v3"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
)
|
||||
|
||||
// APIConfig 节点API配置
|
||||
@@ -17,7 +17,7 @@ type APIConfig struct {
|
||||
}
|
||||
|
||||
func LoadAPIConfig() (*APIConfig, error) {
|
||||
data, err := ioutil.ReadFile(Tea.ConfigFile("api.yaml"))
|
||||
data, err := os.ReadFile(Tea.ConfigFile("api.yaml"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -37,6 +37,6 @@ func (this *APIConfig) WriteFile(path string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioutil.WriteFile(path, data, 0666)
|
||||
err = os.WriteFile(path, data, 0666)
|
||||
return err
|
||||
}
|
||||
|
||||
117
internal/conns/map.go
Normal file
117
internal/conns/map.go
Normal file
@@ -0,0 +1,117 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package conns
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var SharedMap = NewMap()
|
||||
|
||||
type Map struct {
|
||||
m map[string]map[int]net.Conn // ip => { port => Conn }
|
||||
|
||||
locker sync.RWMutex
|
||||
}
|
||||
|
||||
func NewMap() *Map {
|
||||
return &Map{
|
||||
m: map[string]map[int]net.Conn{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) Add(conn net.Conn) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var ip = tcpAddr.IP.String()
|
||||
var port = tcpAddr.Port
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
connMap, ok := this.m[ip]
|
||||
if !ok {
|
||||
this.m[ip] = map[int]net.Conn{
|
||||
port: conn,
|
||||
}
|
||||
} else {
|
||||
connMap[port] = conn
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) Remove(conn net.Conn) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var ip = tcpAddr.IP.String()
|
||||
var port = tcpAddr.Port
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
connMap, ok := this.m[ip]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(connMap, port)
|
||||
|
||||
if len(connMap) == 0 {
|
||||
delete(this.m, ip)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) CountIPConns(ip string) int {
|
||||
this.locker.RLock()
|
||||
var l = len(this.m[ip])
|
||||
this.locker.RUnlock()
|
||||
return l
|
||||
}
|
||||
|
||||
func (this *Map) CloseIPConns(ip string) {
|
||||
var conns = []net.Conn{}
|
||||
|
||||
this.locker.RLock()
|
||||
connMap, ok := this.m[ip]
|
||||
|
||||
// 复制,防止在Close时产生并发冲突
|
||||
if ok {
|
||||
for _, conn := range connMap {
|
||||
conns = append(conns, conn)
|
||||
}
|
||||
}
|
||||
|
||||
// 需要在Close之前结束,防止死循环
|
||||
this.locker.RUnlock()
|
||||
|
||||
if ok {
|
||||
for _, conn := range conns {
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
// 这里不需要从 m 中删除,因为关闭时会自然触发回调
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) AllConns() []net.Conn {
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
var result = []net.Conn{}
|
||||
for _, m := range this.m {
|
||||
for _, conn := range m {
|
||||
result = append(result, conn)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "0.4.10"
|
||||
Version = "0.5.2"
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
package teaconst
|
||||
|
||||
import "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"os"
|
||||
)
|
||||
|
||||
var (
|
||||
// 流量统计
|
||||
@@ -12,6 +15,7 @@ var (
|
||||
|
||||
NodeId int64 = 0
|
||||
NodeIdString = ""
|
||||
IsDaemon = len(os.Args) > 1 && os.Args[1] == "daemon"
|
||||
|
||||
GlobalProductName = nodeconfigs.DefaultProductName
|
||||
|
||||
|
||||
@@ -195,14 +195,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// new connections rate
|
||||
var newConnectionsRate = tcpConfig.NewConnectionsRate
|
||||
if newConnectionsRate <= 0 {
|
||||
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
|
||||
if newConnectionsRate <= 0 {
|
||||
newConnectionsRate = 100000
|
||||
// new connections rate (minutely)
|
||||
var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
|
||||
if newConnectionsMinutelyRate <= 0 {
|
||||
newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
|
||||
if newConnectionsMinutelyRate <= 0 {
|
||||
newConnectionsMinutelyRate = 100000
|
||||
}
|
||||
}
|
||||
var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
|
||||
if newConnectionsMinutelyRateBlockTimeout < 0 {
|
||||
newConnectionsMinutelyRateBlockTimeout = 0
|
||||
}
|
||||
|
||||
// new connections rate (secondly)
|
||||
var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
|
||||
if newConnectionsSecondlyRate <= 0 {
|
||||
newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
|
||||
if newConnectionsSecondlyRate <= 0 {
|
||||
newConnectionsSecondlyRate = 10000
|
||||
}
|
||||
}
|
||||
var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
|
||||
if newConnectionsSecondlyRateBlockTimeout < 0 {
|
||||
newConnectionsSecondlyRateBlockTimeout = 0
|
||||
}
|
||||
|
||||
// 检查是否有变化
|
||||
var hasChanges = false
|
||||
@@ -215,7 +232,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
hasChanges = true
|
||||
break
|
||||
}
|
||||
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
|
||||
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
|
||||
hasChanges = true
|
||||
break
|
||||
}
|
||||
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
|
||||
hasChanges = true
|
||||
break
|
||||
}
|
||||
@@ -251,6 +272,7 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO 让用户选择是drop还是reject
|
||||
if maxConnectionsPerIP > 0 {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
@@ -261,14 +283,47 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
}
|
||||
}
|
||||
|
||||
if newConnectionsRate > 0 {
|
||||
// TODO 思考是否有惩罚机制
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
// 超过一定速率就drop或者加入黑名单(分钟)
|
||||
// TODO 让用户选择是drop还是reject
|
||||
if newConnectionsMinutelyRate > 0 {
|
||||
if newConnectionsMinutelyRateBlockTimeout > 0 {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
} else {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 超过一定速率就drop或者加入黑名单(秒)
|
||||
// TODO 让用户选择是drop还是reject
|
||||
if newConnectionsSecondlyRate > 0 {
|
||||
if newConnectionsSecondlyRateBlockTimeout > 0 {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
} else {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -335,14 +390,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
|
||||
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
|
||||
for _, rule := range rules {
|
||||
var pieces = this.decodeUserData(rule.UserData())
|
||||
if len(pieces) != 4 {
|
||||
if len(pieces) < 4 {
|
||||
continue
|
||||
}
|
||||
if pieces[0] != "tcp" {
|
||||
continue
|
||||
}
|
||||
switch pieces[2] {
|
||||
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
|
||||
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
|
||||
err := chain.DeleteRule(rule)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
47
internal/firewalls/firewall_base.go
Normal file
47
internal/firewalls/firewall_base.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BaseFirewall struct {
|
||||
locker sync.Mutex
|
||||
latestIPTimes []string // [ip@time, ....]
|
||||
}
|
||||
|
||||
// 检查是否在最近添加过
|
||||
func (this *BaseFirewall) checkLatestIP(ip string) bool {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var expiredIndex = -1
|
||||
for index, ipTime := range this.latestIPTimes {
|
||||
var pieces = strings.Split(ipTime, "@")
|
||||
var oldIP = pieces[0]
|
||||
var oldTimestamp = pieces[1]
|
||||
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
|
||||
expiredIndex = index
|
||||
continue
|
||||
}
|
||||
if oldIP == ip {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if expiredIndex > -1 {
|
||||
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
|
||||
}
|
||||
|
||||
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
|
||||
const maxLen = 128
|
||||
if len(this.latestIPTimes) > maxLen {
|
||||
this.latestIPTimes = this.latestIPTimes[1:]
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -3,6 +3,8 @@
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
@@ -10,15 +12,22 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type firewalldCmd struct {
|
||||
cmd *exec.Cmd
|
||||
denyIP string
|
||||
}
|
||||
|
||||
type Firewalld struct {
|
||||
BaseFirewall
|
||||
|
||||
isReady bool
|
||||
exe string
|
||||
cmdQueue chan *exec.Cmd
|
||||
cmdQueue chan *firewalldCmd
|
||||
}
|
||||
|
||||
func NewFirewalld() *Firewalld {
|
||||
var firewalld = &Firewalld{
|
||||
cmdQueue: make(chan *exec.Cmd, 4096),
|
||||
cmdQueue: make(chan *firewalldCmd, 4096),
|
||||
}
|
||||
|
||||
path, err := exec.LookPath("firewall-cmd")
|
||||
@@ -40,13 +49,19 @@ func NewFirewalld() *Firewalld {
|
||||
|
||||
func (this *Firewalld) init() {
|
||||
goman.New(func() {
|
||||
for cmd := range this.cmdQueue {
|
||||
for c := range this.cmdQueue {
|
||||
var cmd = c.cmd
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
if strings.HasPrefix(err.Error(), "Warning:") {
|
||||
continue
|
||||
}
|
||||
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
|
||||
} else {
|
||||
// 关闭连接
|
||||
if len(c.denyIP) > 0 {
|
||||
conns.SharedMap.CloseIPConns(c.denyIP)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -71,7 +86,7 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
|
||||
return nil
|
||||
}
|
||||
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -81,12 +96,12 @@ func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--add-port="+port, "--permanent")
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--add-port="+port)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +113,7 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
|
||||
return nil
|
||||
}
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -107,12 +122,12 @@ func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol str
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+port, "--permanent")
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+port)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -130,6 +145,12 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 避免短时间内重复添加
|
||||
if this.checkLatestIP(ip) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
@@ -139,14 +160,20 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
|
||||
}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 避免短时间内重复添加
|
||||
if async && this.checkLatestIP(ip) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
@@ -156,7 +183,18 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
|
||||
}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
if async {
|
||||
this.pushCmd(cmd, ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 关闭连接
|
||||
defer conns.SharedMap.CloseIPConns(ip)
|
||||
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -164,6 +202,7 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
@@ -171,14 +210,14 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
|
||||
for _, action := range []string{"reject", "drop"} {
|
||||
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
|
||||
func (this *Firewalld) pushCmd(cmd *exec.Cmd, denyIP string) {
|
||||
select {
|
||||
case this.cmdQueue <- cmd:
|
||||
case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
|
||||
default:
|
||||
// we discard the command
|
||||
}
|
||||
|
||||
@@ -23,7 +23,10 @@ type FirewallInterface interface {
|
||||
RejectSourceIP(ip string, timeoutSeconds int) error
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
DropSourceIP(ip string, timeoutSeconds int) error
|
||||
// ip 要封禁的IP
|
||||
// timeoutSeconds 过期时间
|
||||
// async 是否异步
|
||||
DropSourceIP(ip string, timeoutSeconds int, async bool) error
|
||||
|
||||
// RemoveSourceIP 删除某个源IP
|
||||
RemoveSourceIP(ip string) error
|
||||
|
||||
@@ -47,7 +47,7 @@ func (this *MockFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
}
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
|
||||
_ = ip
|
||||
_ = timeoutSeconds
|
||||
return nil
|
||||
|
||||
@@ -7,8 +7,11 @@ package firewalls
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net"
|
||||
@@ -21,9 +24,13 @@ import (
|
||||
|
||||
// check nft status, if being enabled we load it automatically
|
||||
func init() {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
if runtime.GOOS == "linux" {
|
||||
var ticker = time.NewTicker(3 * time.Minute)
|
||||
go func() {
|
||||
goman.New(func() {
|
||||
for range ticker.C {
|
||||
// if already ready, we break
|
||||
if nftablesIsReady {
|
||||
@@ -48,7 +55,7 @@ func init() {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,9 +81,16 @@ func (this *nftablesTableDefinition) protocol() string {
|
||||
return "ip"
|
||||
}
|
||||
|
||||
type blockIPItem struct {
|
||||
action string
|
||||
ip string
|
||||
timeoutSeconds int
|
||||
}
|
||||
|
||||
func NewNFTablesFirewall() (*NFTablesFirewall, error) {
|
||||
var firewall = &NFTablesFirewall{
|
||||
conn: nftables.NewConn(),
|
||||
conn: nftables.NewConn(),
|
||||
dropIPQueue: make(chan *blockIPItem, 4096),
|
||||
}
|
||||
err := firewall.init()
|
||||
if err != nil {
|
||||
@@ -87,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
|
||||
}
|
||||
|
||||
type NFTablesFirewall struct {
|
||||
BaseFirewall
|
||||
|
||||
conn *nftables.Conn
|
||||
isReady bool
|
||||
version string
|
||||
@@ -98,6 +114,8 @@ type NFTablesFirewall struct {
|
||||
denyIPv6Set *nftables.Set
|
||||
|
||||
firewalld *Firewalld
|
||||
|
||||
dropIPQueue chan *blockIPItem
|
||||
}
|
||||
|
||||
func (this *NFTablesFirewall) init() error {
|
||||
@@ -243,6 +261,18 @@ func (this *NFTablesFirewall) init() error {
|
||||
nftablesIsReady = true
|
||||
nftablesInstance = this
|
||||
|
||||
goman.New(func() {
|
||||
for ipItem := range this.dropIPQueue {
|
||||
switch ipItem.action {
|
||||
case "drop":
|
||||
err = this.DropSourceIP(ipItem.ip, ipItem.timeoutSeconds, false)
|
||||
if err != nil {
|
||||
remotelogs.Warn("NFTABLES", "drop ip '"+ipItem.ip+"' failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// load firewalld
|
||||
var firewalld = NewFirewalld()
|
||||
if firewalld.IsReady() {
|
||||
@@ -307,16 +337,40 @@ func (this *NFTablesFirewall) AllowSourceIP(ip string) error {
|
||||
// RejectSourceIP 拒绝某个源IP连接
|
||||
// we did not create set for drop ip, so we reuse DropSourceIP() method here
|
||||
func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
return this.DropSourceIP(ip, timeoutSeconds)
|
||||
return this.DropSourceIP(ip, timeoutSeconds, true)
|
||||
}
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
|
||||
var data = net.ParseIP(ip)
|
||||
if data == nil {
|
||||
return errors.New("invalid ip '" + ip + "'")
|
||||
}
|
||||
|
||||
// 尝试关闭连接
|
||||
conns.SharedMap.CloseIPConns(ip)
|
||||
|
||||
// 避免短时间内重复添加
|
||||
if async && this.checkLatestIP(ip) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if async {
|
||||
select {
|
||||
case this.dropIPQueue <- &blockIPItem{
|
||||
action: "drop",
|
||||
ip: ip,
|
||||
timeoutSeconds: timeoutSeconds,
|
||||
}:
|
||||
default:
|
||||
return errors.New("drop ip queue is full")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 再次尝试关闭连接
|
||||
defer conns.SharedMap.CloseIPConns(ip)
|
||||
|
||||
if strings.Contains(ip, ":") { // ipv6
|
||||
if this.denyIPv6Set == nil {
|
||||
return errors.New("ipv6 ip set is nil")
|
||||
@@ -393,3 +447,35 @@ func (this *NFTablesFirewall) readVersion(nftPath string) string {
|
||||
}
|
||||
return versionMatches[1]
|
||||
}
|
||||
|
||||
// 检查是否在最近添加过
|
||||
func (this *NFTablesFirewall) existLatestIP(ip string) bool {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var expiredIndex = -1
|
||||
for index, ipTime := range this.latestIPTimes {
|
||||
var pieces = strings.Split(ipTime, "@")
|
||||
var oldIP = pieces[0]
|
||||
var oldTimestamp = pieces[1]
|
||||
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
|
||||
expiredIndex = index
|
||||
continue
|
||||
}
|
||||
if oldIP == ip {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if expiredIndex > -1 {
|
||||
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
|
||||
}
|
||||
|
||||
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
|
||||
const maxLen = 128
|
||||
if len(this.latestIPTimes) > maxLen {
|
||||
this.latestIPTimes = this.latestIPTimes[1:]
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) erro
|
||||
}
|
||||
|
||||
// DropSourceIP 丢弃某个源IP数据
|
||||
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
|
||||
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package goman
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -14,6 +15,10 @@ var instanceId = uint64(0)
|
||||
|
||||
// New 新创建goroutine
|
||||
func New(f func()) {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
|
||||
go func() {
|
||||
@@ -42,6 +47,10 @@ func New(f func()) {
|
||||
|
||||
// NewWithArgs 创建带有参数的goroutine
|
||||
func NewWithArgs(f func(args ...interface{}), args ...interface{}) {
|
||||
if teaconst.IsDaemon {
|
||||
return
|
||||
}
|
||||
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
|
||||
go func() {
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
// 源码改自:https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
|
||||
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
IndexBlockLength = 12
|
||||
)
|
||||
|
||||
var err error
|
||||
|
||||
type IP2Region struct {
|
||||
headerSip []int64
|
||||
headerPtr []int64
|
||||
headerLen int64
|
||||
|
||||
// super block index info
|
||||
firstIndexPtr int64
|
||||
lastIndexPtr int64
|
||||
totalBlocks int64
|
||||
|
||||
dbData []byte
|
||||
}
|
||||
|
||||
type IpInfo struct {
|
||||
CityId int64
|
||||
Country string
|
||||
Region string
|
||||
Province string
|
||||
City string
|
||||
ISP string
|
||||
}
|
||||
|
||||
func (ip IpInfo) String() string {
|
||||
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
|
||||
}
|
||||
|
||||
func getIpInfo(cityId int64, line []byte) *IpInfo {
|
||||
lineSlice := strings.Split(string(line), "|")
|
||||
ipInfo := &IpInfo{}
|
||||
length := len(lineSlice)
|
||||
ipInfo.CityId = cityId
|
||||
if length < 5 {
|
||||
for i := 0; i <= 5-length; i++ {
|
||||
lineSlice = append(lineSlice, "")
|
||||
}
|
||||
}
|
||||
|
||||
ipInfo.Country = lineSlice[0]
|
||||
ipInfo.Region = lineSlice[1]
|
||||
ipInfo.Province = lineSlice[2]
|
||||
ipInfo.City = lineSlice[3]
|
||||
ipInfo.ISP = lineSlice[4]
|
||||
return ipInfo
|
||||
}
|
||||
|
||||
func NewIP2Region(path string) (*IP2Region, error) {
|
||||
var region = &IP2Region{}
|
||||
region.dbData, err = ioutil.ReadFile(path)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
region.firstIndexPtr = region.ipLongAtOffset(0)
|
||||
region.lastIndexPtr = region.ipLongAtOffset(4)
|
||||
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
|
||||
return region, nil
|
||||
}
|
||||
|
||||
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
|
||||
ip, err := ip2long(ipStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
h := this.totalBlocks
|
||||
var dataPtr, l int64
|
||||
for l <= h {
|
||||
m := (l + h) >> 1
|
||||
p := this.firstIndexPtr + m*IndexBlockLength
|
||||
sip := this.ipLongAtOffset(p)
|
||||
if ip < sip {
|
||||
h = m - 1
|
||||
} else {
|
||||
eip := this.ipLongAtOffset(p + 4)
|
||||
if ip > eip {
|
||||
l = m + 1
|
||||
} else {
|
||||
dataPtr = this.ipLongAtOffset(p + 8)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if dataPtr == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
dataLen := (dataPtr >> 24) & 0xFF
|
||||
dataPtr = dataPtr & 0x00FFFFFF
|
||||
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
|
||||
}
|
||||
|
||||
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
|
||||
return int64(this.dbData[offset]) |
|
||||
int64(this.dbData[offset+1])<<8 |
|
||||
int64(this.dbData[offset+2])<<16 |
|
||||
int64(this.dbData[offset+3])<<24
|
||||
}
|
||||
|
||||
func ip2long(IpStr string) (int64, error) {
|
||||
bits := strings.Split(IpStr, ".")
|
||||
if len(bits) != 4 {
|
||||
return 0, errors.New("ip format error")
|
||||
}
|
||||
|
||||
var sum int64
|
||||
for i, n := range bits {
|
||||
bit, _ := strconv.ParseInt(n, 10, 64)
|
||||
sum += bit << uint(24-8*i)
|
||||
}
|
||||
|
||||
return sum, nil
|
||||
}
|
||||
@@ -18,7 +18,8 @@ import (
|
||||
type IPListDB struct {
|
||||
db *sql.DB
|
||||
|
||||
itemTableName string
|
||||
itemTableName string
|
||||
|
||||
deleteExpiredItemsStmt *sql.Stmt
|
||||
deleteItemStmt *sql.Stmt
|
||||
insertItemStmt *sql.Stmt
|
||||
@@ -53,7 +54,11 @@ func (this *IPListDB) init() error {
|
||||
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
var path = this.dir + "/ip_list.db"
|
||||
_ = os.Remove(path + "-shm")
|
||||
_ = os.Remove(path + "-wal")
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -194,12 +199,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
row := this.selectMaxVersionStmt.QueryRow()
|
||||
var row = this.selectMaxVersionStmt.QueryRow()
|
||||
if row == nil {
|
||||
return 0
|
||||
}
|
||||
var version int64
|
||||
err = row.Scan(&version)
|
||||
err := row.Scan(&version)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = db.AddItem(&pb.IPItem{
|
||||
Id: 1,
|
||||
IpFrom: "192.168.1.101",
|
||||
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = db.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
type LibraryInterface interface {
|
||||
// Load 加载数据库文件
|
||||
Load(dbPath string) error
|
||||
|
||||
// Lookup 查询IP
|
||||
// 返回结果有可能为空
|
||||
Lookup(ip string) (*Result, error)
|
||||
|
||||
// Close 关闭数据库文件
|
||||
Close()
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type IP2RegionLibrary struct {
|
||||
db *IP2Region
|
||||
}
|
||||
|
||||
func (this *IP2RegionLibrary) Load(dbPath string) error {
|
||||
db, err := NewIP2Region(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.db = db
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
|
||||
// 暂不支持IPv6
|
||||
if strings.Contains(ip, ":") {
|
||||
return nil, nil
|
||||
}
|
||||
if net.ParseIP(ip) == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if this.db == nil {
|
||||
return nil, errors.New("library has not been loaded")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// 防止panic发生
|
||||
err := recover()
|
||||
if err != nil {
|
||||
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
|
||||
}
|
||||
}()
|
||||
|
||||
info, err := this.db.MemorySearch(ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if info.Country == "0" {
|
||||
info.Country = ""
|
||||
}
|
||||
if info.Region == "0" {
|
||||
info.Region = ""
|
||||
}
|
||||
if info.Province == "0" {
|
||||
info.Province = ""
|
||||
}
|
||||
if info.City == "0" {
|
||||
info.City = ""
|
||||
}
|
||||
if info.ISP == "0" {
|
||||
info.ISP = ""
|
||||
}
|
||||
|
||||
return &Result{
|
||||
CityId: info.CityId,
|
||||
Country: info.Country,
|
||||
Region: info.Region,
|
||||
Province: info.Province,
|
||||
City: info.City,
|
||||
ISP: info.ISP,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *IP2RegionLibrary) Close() {
|
||||
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
|
||||
var mem = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(mem)
|
||||
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var mem2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(mem2)
|
||||
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, ip := range []string{"8.8.9.9"} {
|
||||
result, err := library.Lookup(ip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("IP:", ip, "result:", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Lookup(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
|
||||
result, err := library.Lookup(ip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("IP:", ip, "result:", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var count = 4000
|
||||
var wg = sync.WaitGroup{}
|
||||
wg.Add(count)
|
||||
for i := 0; i < count; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Memory(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
before := time.Now()
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
|
||||
}
|
||||
|
||||
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var library = &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = library.Lookup("8.8.8.8")
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/files"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var SharedManager = NewManager()
|
||||
var SharedLibrary LibraryInterface
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
// 初始化
|
||||
library, err := SharedManager.Load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("IP_LIBRARY", err)
|
||||
return
|
||||
}
|
||||
SharedLibrary = library
|
||||
})
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
code string
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
return &Manager{}
|
||||
}
|
||||
|
||||
func (this *Manager) Load() (LibraryInterface, error) {
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config := nodeConfig.GlobalConfig
|
||||
if config == nil {
|
||||
config = &serverconfigs.GlobalConfig{}
|
||||
}
|
||||
|
||||
// 当前正在使用的IP库代号
|
||||
code := config.IPLibrary.Code
|
||||
if len(code) == 0 {
|
||||
code = serverconfigs.DefaultIPLibraryType
|
||||
}
|
||||
|
||||
dir := Tea.Root + "/resources/ipdata/" + code
|
||||
var lastVersion int64 = -1
|
||||
lastFilename := ""
|
||||
for _, file := range files.NewFile(dir).List() {
|
||||
filename := file.Name()
|
||||
|
||||
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
|
||||
if reg.MatchString(filename) { // 先查找有版本号的
|
||||
result := reg.FindStringSubmatch(filename)
|
||||
version := types.Int64(result[1])
|
||||
if version > lastVersion {
|
||||
lastVersion = version
|
||||
lastFilename = filename
|
||||
}
|
||||
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
|
||||
if lastVersion == -1 {
|
||||
lastFilename = filename
|
||||
lastVersion = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(lastFilename) == 0 {
|
||||
return nil, errors.New("ip library file not found")
|
||||
}
|
||||
|
||||
var libraryPtr LibraryInterface
|
||||
switch code {
|
||||
case serverconfigs.IPLibraryTypeIP2Region:
|
||||
libraryPtr = &IP2RegionLibrary{}
|
||||
default:
|
||||
return nil, errors.New("invalid ip library code '" + code + "'")
|
||||
}
|
||||
|
||||
err = libraryPtr.Load(dir + "/" + lastFilename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return libraryPtr, nil
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedCityManager = NewCityManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedCityManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedCityManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// CityManager 中国省份信息管理
|
||||
type CityManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
cityMap map[string]int64 // provinceName_cityName => cityName
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewCityManager() *CityManager {
|
||||
return &CityManager{
|
||||
cacheFile: Tea.Root + "/configs/region_city.json.cache",
|
||||
cityMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("CITY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("City_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("CITY_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
|
||||
this.locker.RLock()
|
||||
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *CityManager) load() error {
|
||||
data, err := ioutil.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.cityMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新城市信息
|
||||
func (this *CityManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionCityRPC().FindAllEnabledRegionCities(rpcClient.Context(), &pb.FindAllEnabledRegionCitiesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, city := range resp.RegionCities {
|
||||
for _, code := range city.Codes {
|
||||
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.cityMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = ioutil.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package iplibrary
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewCityManager(t *testing.T) {
|
||||
var manager = NewCityManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup(16, "许昌市"))
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedCountryManager = NewCountryManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedCountryManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedCountryManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// CountryManager 国家/地区信息管理
|
||||
type CountryManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
countryMap map[string]int64 // countryName => countryId
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewCountryManager() *CountryManager {
|
||||
return &CountryManager{
|
||||
cacheFile: Tea.Root + "/configs/region_country.json.cache",
|
||||
countryMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
|
||||
this.locker.RLock()
|
||||
countryId, _ = this.countryMap[countryName]
|
||||
this.locker.RUnlock()
|
||||
return countryId
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *CountryManager) load() error {
|
||||
data, err := ioutil.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.countryMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新国家信息
|
||||
func (this *CountryManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionCountryRPC().FindAllEnabledRegionCountries(rpcClient.Context(), &pb.FindAllEnabledRegionCountriesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, country := range resp.RegionCountries {
|
||||
for _, code := range country.Codes {
|
||||
m[code] = country.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.countryMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
err = ioutil.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCountryManager_load(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.countryMap)
|
||||
}
|
||||
|
||||
func TestCountryManager_loop(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.countryMap)
|
||||
}
|
||||
|
||||
func TestCountryManager_loop_skip(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
for i := 0; i < 10; i++ {
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountryManager_Lookup(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
|
||||
}
|
||||
|
||||
func BenchmarkCountryManager_Lookup(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
manager := NewCountryManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = manager.Lookup("中国")
|
||||
}
|
||||
}
|
||||
@@ -43,7 +43,7 @@ type IPListManager struct {
|
||||
|
||||
func NewIPListManager() *IPListManager {
|
||||
return &IPListManager{
|
||||
pageSize: 500,
|
||||
pageSize: 1000,
|
||||
listMap: map[int64]*IPList{},
|
||||
}
|
||||
}
|
||||
@@ -111,15 +111,19 @@ func (this *IPListManager) init() {
|
||||
var size int64 = 1000
|
||||
for {
|
||||
items, err := db.ReadItems(offset, size)
|
||||
var l = len(items)
|
||||
if err != nil {
|
||||
remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
|
||||
} else {
|
||||
if len(items) == 0 {
|
||||
if l == 0 {
|
||||
break
|
||||
}
|
||||
this.processItems(items, false)
|
||||
if int64(l) < size {
|
||||
break
|
||||
}
|
||||
}
|
||||
offset += int64(len(items))
|
||||
offset += int64(l)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -144,7 +148,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
|
||||
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
|
||||
Version: this.version,
|
||||
Size: this.pageSize,
|
||||
})
|
||||
@@ -155,7 +159,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
items := itemsResp.IpItems
|
||||
var items = itemsResp.IpItems
|
||||
if len(items) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
@@ -183,7 +187,6 @@ func (this *IPListManager) FindList(listId int64) *IPList {
|
||||
}
|
||||
|
||||
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
|
||||
this.locker.Lock()
|
||||
var changedLists = map[*IPList]zero.Zero{}
|
||||
for _, item := range items {
|
||||
var list *IPList
|
||||
@@ -203,11 +206,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
|
||||
list = GlobalWhiteIPList
|
||||
}
|
||||
} else { // 其他List
|
||||
this.locker.Lock()
|
||||
list = this.listMap[item.ListId]
|
||||
this.locker.Unlock()
|
||||
}
|
||||
if list == nil {
|
||||
list = NewIPList()
|
||||
this.locker.Lock()
|
||||
this.listMap[item.ListId] = list
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
changedLists[list] = zero.New()
|
||||
@@ -246,8 +253,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
|
||||
changedList.Sort()
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
|
||||
if fromRemote {
|
||||
var latestVersion = items[len(items)-1].Version
|
||||
if latestVersion > this.version {
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedProviderManager = NewProviderManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedProviderManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedProviderManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// ProviderManager 中国省份信息管理
|
||||
type ProviderManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
providerMap map[string]int64 // name => id
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewProviderManager() *ProviderManager {
|
||||
return &ProviderManager{
|
||||
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
|
||||
providerMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
|
||||
this.locker.RLock()
|
||||
providerId, _ = this.providerMap[providerName]
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *ProviderManager) load() error {
|
||||
data, err := ioutil.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.providerMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新服务商信息
|
||||
func (this *ProviderManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionProviderRPC().FindAllEnabledRegionProviders(rpcClient.Context(), &pb.FindAllEnabledRegionProvidersRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, provider := range resp.RegionProviders {
|
||||
for _, code := range provider.Codes {
|
||||
m[code] = provider.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.providerMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = ioutil.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package iplibrary
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewProviderManager(t *testing.T) {
|
||||
var manager = NewProviderManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("阿里云"))
|
||||
t.Log(manager.Lookup("阿里云2"))
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ChinaCountryId int64 = 1
|
||||
)
|
||||
|
||||
var SharedProvinceManager = NewProvinceManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedProvinceManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedProvinceManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// ProvinceManager 中国省份信息管理
|
||||
type ProvinceManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
provinceMap map[string]int64 // provinceName => provinceId
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewProvinceManager() *ProvinceManager {
|
||||
return &ProvinceManager{
|
||||
cacheFile: Tea.Root + "/configs/region_province.json.cache",
|
||||
provinceMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
|
||||
this.locker.RLock()
|
||||
provinceId, _ = this.provinceMap[provinceName]
|
||||
this.locker.RUnlock()
|
||||
return provinceId
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *ProvinceManager) load() error {
|
||||
data, err := ioutil.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.provinceMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新省份信息
|
||||
func (this *ProvinceManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionProvinceRPC().FindAllEnabledRegionProvincesWithCountryId(rpcClient.Context(), &pb.FindAllEnabledRegionProvincesWithCountryIdRequest{
|
||||
RegionCountryId: ChinaCountryId,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, province := range resp.RegionProvinces {
|
||||
for _, code := range province.Codes {
|
||||
m[code] = province.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.provinceMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = ioutil.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProvinceManager_load(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.provinceMap)
|
||||
}
|
||||
|
||||
func TestProvinceManager_loop(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.provinceMap)
|
||||
}
|
||||
|
||||
func TestProvinceManager_loop_skip(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
for i := 0; i < 10; i++ {
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvinceManager_Lookup(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
|
||||
}
|
||||
|
||||
func BenchmarkProvinceManager_Lookup(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
manager := NewProvinceManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = manager.Lookup("安徽省")
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManager_Load(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
manager := NewManager()
|
||||
lib, err := manager.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(lib.Lookup("1.2.3.4"))
|
||||
t.Log(lib.Lookup("2.3.4.5"))
|
||||
t.Log(lib.Lookup("200.200.200.200"))
|
||||
t.Log(lib.Lookup("202.106.0.20"))
|
||||
}
|
||||
|
||||
func TestNewManager(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
t.Log(SharedLibrary)
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedUpdater = NewUpdater()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
goman.New(func() {
|
||||
SharedUpdater.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedUpdater.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// Updater IP库更新程序
|
||||
type Updater struct {
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
// NewUpdater 获取新对象
|
||||
func NewUpdater() *Updater {
|
||||
return &Updater{}
|
||||
}
|
||||
|
||||
// Start 开始更新
|
||||
func (this *Updater) Start() {
|
||||
// 这里不需要太频繁检查更新,因为通常不需要更新IP库
|
||||
this.ticker = time.NewTicker(1 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("IP_LIBRARY", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Updater) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// 单次任务
|
||||
func (this *Updater) loop() error {
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if nodeConfig.GlobalConfig == nil {
|
||||
return nil
|
||||
}
|
||||
code := nodeConfig.GlobalConfig.IPLibrary.Code
|
||||
if len(code) == 0 {
|
||||
code = serverconfigs.DefaultIPLibraryType
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lib := libraryResp.IpLibrary
|
||||
if lib == nil || lib.File == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
typeInfo := serverconfigs.FindIPLibraryWithType(code)
|
||||
if typeInfo == nil {
|
||||
return errors.New("invalid ip library code '" + code + "'")
|
||||
}
|
||||
|
||||
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
|
||||
|
||||
// 是否已经存在
|
||||
_, err = os.Stat(path)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 开始下载
|
||||
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunkIds := fileChunkIdsResp.FileChunkIds
|
||||
if len(chunkIds) == 0 {
|
||||
return nil
|
||||
}
|
||||
isOk := false
|
||||
|
||||
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// 如果保存不成功就直接删除
|
||||
if !isOk {
|
||||
_ = fp.Close()
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
for _, chunkId := range chunkIds {
|
||||
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunk := chunkResp.FileChunk
|
||||
|
||||
if chunk == nil {
|
||||
continue
|
||||
}
|
||||
_, err = fp.Write(chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = fp.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 重新加载
|
||||
library, err := SharedManager.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
SharedLibrary = library
|
||||
|
||||
isOk = true
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpdater_loop(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
updater := NewUpdater()
|
||||
err := updater.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package metrics
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"strconv"
|
||||
"sync"
|
||||
@@ -11,7 +12,15 @@ import (
|
||||
|
||||
var SharedManager = NewManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedManager.Quit()
|
||||
})
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
isQuiting bool
|
||||
|
||||
tasks map[int64]*Task // itemId => *Task
|
||||
categoryTasks map[string][]*Task // category => []*Task
|
||||
locker sync.RWMutex
|
||||
@@ -29,6 +38,10 @@ func NewManager() *Manager {
|
||||
}
|
||||
|
||||
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
|
||||
if this.isQuiting {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
@@ -101,6 +114,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
|
||||
|
||||
// Add 添加数据
|
||||
func (this *Manager) Add(obj MetricInterface) {
|
||||
if this.isQuiting {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.RLock()
|
||||
for _, task := range this.categoryTasks[obj.MetricCategory()] {
|
||||
task.Add(obj)
|
||||
@@ -119,3 +136,17 @@ func (this *Manager) HasTCPMetrics() bool {
|
||||
func (this *Manager) HasUDPMetrics() bool {
|
||||
return this.hasUDPMetrics
|
||||
}
|
||||
|
||||
// Quit 退出管理器
|
||||
func (this *Manager) Quit() {
|
||||
this.isQuiting = true
|
||||
|
||||
remotelogs.Println("METRIC_MANAGER", "quit")
|
||||
|
||||
this.locker.Lock()
|
||||
for _, task := range this.tasks {
|
||||
_ = task.Stop()
|
||||
}
|
||||
this.tasks = map[int64]*Task{}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"os"
|
||||
"strconv"
|
||||
@@ -23,18 +24,19 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const MaxQueueSize = 2048
|
||||
const MaxQueueSize = 256 // TODO 可以配置,可以在单个任务里配置
|
||||
|
||||
// Task 单个指标任务
|
||||
// 数据库存储:
|
||||
// data/
|
||||
// metric.$ID.db
|
||||
// stats
|
||||
// id, keys, value, time, serverId, hash
|
||||
// 原理:
|
||||
// 添加或者有变更时 isUploaded = false
|
||||
// 上传时检查 isUploaded 状态
|
||||
// 只上传每个服务中排序最前面的 N 个数据
|
||||
//
|
||||
// data/
|
||||
// metric.$ID.db
|
||||
// stats
|
||||
// id, keys, value, time, serverId, hash
|
||||
// 原理:
|
||||
// 添加或者有变更时 isUploaded = false
|
||||
// 上传时检查 isUploaded 状态
|
||||
// 只上传每个服务中排序最前面的 N 个数据
|
||||
type Task struct {
|
||||
item *serverconfigs.MetricItemConfig
|
||||
isLoaded bool
|
||||
@@ -88,7 +90,11 @@ func (this *Task) Init() error {
|
||||
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
|
||||
_ = os.Remove(path + "-shm")
|
||||
_ = os.Remove(path + "-wal")
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -372,7 +378,9 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
|
||||
for _, serverId := range serverIds {
|
||||
for _, currentTime := range times {
|
||||
idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
|
||||
var t = trackers.Begin("[METRIC]SELECT_TOP_STMT")
|
||||
rows, err := this.selectTopStmt.Query(serverId, this.item.Version, currentTime)
|
||||
t.End()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -421,7 +429,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
|
||||
_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
|
||||
MetricStats: pbStats,
|
||||
Time: currentTime,
|
||||
ServerId: serverId,
|
||||
|
||||
@@ -69,7 +69,7 @@ func (this *ValueQueue) Loop() error {
|
||||
}
|
||||
|
||||
for value := range this.valuesChan {
|
||||
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
|
||||
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
|
||||
Item: value.Item,
|
||||
ValueJSON: value.ValueJSON,
|
||||
CreatedAt: value.CreatedAt,
|
||||
|
||||
@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
|
||||
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
|
||||
if err != nil {
|
||||
statusErr, ok:= status.FromError(err)
|
||||
statusErr, ok := status.FromError(err)
|
||||
if ok {
|
||||
logs.Println(statusErr.Code())
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func (this *APIStream) loop() error {
|
||||
cancelFunc()
|
||||
}()
|
||||
|
||||
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
|
||||
nodeStream, err := rpcClient.NodeRPC.NodeStream(ctx)
|
||||
if err != nil {
|
||||
if this.isQuiting {
|
||||
return nil
|
||||
|
||||
@@ -5,12 +5,14 @@ package nodes
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net"
|
||||
"os"
|
||||
@@ -22,6 +24,8 @@ import (
|
||||
|
||||
// ClientConn 客户端连接
|
||||
type ClientConn struct {
|
||||
BaseClientConn
|
||||
|
||||
once sync.Once
|
||||
|
||||
isTLS bool
|
||||
@@ -31,29 +35,28 @@ type ClientConn struct {
|
||||
isLO bool // 是否为环路
|
||||
|
||||
hasResetSYNFlood bool
|
||||
|
||||
BaseClientConn
|
||||
}
|
||||
|
||||
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
|
||||
if quickClose {
|
||||
// TCP
|
||||
tcpConn, ok := conn.(*net.TCPConn)
|
||||
if ok {
|
||||
// TODO 可以在配置中设置此值
|
||||
_ = tcpConn.SetLinger(nodeconfigs.DefaultTCPLinger)
|
||||
}
|
||||
}
|
||||
|
||||
func NewClientConn(rawConn net.Conn, isTLS bool, quickClose bool) net.Conn {
|
||||
// 是否为环路
|
||||
var remoteAddr = conn.RemoteAddr().String()
|
||||
var remoteAddr = rawConn.RemoteAddr().String()
|
||||
var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
|
||||
|
||||
return &ClientConn{
|
||||
BaseClientConn: BaseClientConn{rawConn: conn},
|
||||
var conn = &ClientConn{
|
||||
BaseClientConn: BaseClientConn{rawConn: rawConn},
|
||||
isTLS: isTLS,
|
||||
isLO: isLO,
|
||||
}
|
||||
|
||||
if quickClose {
|
||||
// TODO 可以在配置中设置此值
|
||||
_ = conn.SetLinger(nodeconfigs.DefaultTCPLinger)
|
||||
}
|
||||
|
||||
// 加入到Map
|
||||
conns.SharedMap.Add(conn)
|
||||
|
||||
return conn
|
||||
}
|
||||
|
||||
func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
@@ -110,11 +113,10 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
func (this *ClientConn) Write(b []byte) (n int, err error) {
|
||||
n, err = this.rawConn.Write(b)
|
||||
if n > 0 {
|
||||
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
|
||||
|
||||
// 统计当前服务带宽
|
||||
if this.serverId > 0 {
|
||||
if !this.isLO { // 环路不统计带宽,避免缓存预热等行为产生带宽
|
||||
if !this.isLO || Tea.IsTesting() { // 环路不统计带宽,避免缓存预热等行为产生带宽
|
||||
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
|
||||
stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
|
||||
}
|
||||
}
|
||||
@@ -132,6 +134,9 @@ func (this *ClientConn) Close() error {
|
||||
// 不能加条件限制,因为服务配置随时有变化
|
||||
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
|
||||
|
||||
// 从conn map中移除
|
||||
conns.SharedMap.Remove(this)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -177,6 +182,11 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
|
||||
if timeout <= 0 {
|
||||
timeout = 600
|
||||
}
|
||||
|
||||
// 关闭当前连接
|
||||
_ = this.SetLinger(0)
|
||||
_ = this.Close()
|
||||
|
||||
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击,当前1分钟"+types.String(result)+"次空连接")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
package nodes
|
||||
|
||||
import "net"
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
)
|
||||
|
||||
type BaseClientConn struct {
|
||||
rawConn net.Conn
|
||||
@@ -42,6 +45,17 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
|
||||
// SetServerId 设置服务ID
|
||||
func (this *BaseClientConn) SetServerId(serverId int64) {
|
||||
this.serverId = serverId
|
||||
|
||||
// 设置包装前连接
|
||||
switch conn := this.rawConn.(type) {
|
||||
case *tls.Conn:
|
||||
nativeConn, ok := conn.NetConn().(ClientConnInterface)
|
||||
if ok {
|
||||
nativeConn.SetServerId(serverId)
|
||||
}
|
||||
case *ClientConn:
|
||||
conn.SetServerId(serverId)
|
||||
}
|
||||
}
|
||||
|
||||
// ServerId 读取当前连接绑定的服务ID
|
||||
@@ -52,6 +66,17 @@ func (this *BaseClientConn) ServerId() int64 {
|
||||
// SetUserId 设置所属服务的用户ID
|
||||
func (this *BaseClientConn) SetUserId(userId int64) {
|
||||
this.userId = userId
|
||||
|
||||
// 设置包装前连接
|
||||
switch conn := this.rawConn.(type) {
|
||||
case *tls.Conn:
|
||||
nativeConn, ok := conn.NetConn().(ClientConnInterface)
|
||||
if ok {
|
||||
nativeConn.SetUserId(userId)
|
||||
}
|
||||
case *ClientConn:
|
||||
conn.SetUserId(userId)
|
||||
}
|
||||
}
|
||||
|
||||
// UserId 获取当前连接所属服务的用户ID
|
||||
@@ -66,9 +91,20 @@ func (this *BaseClientConn) RawIP() string {
|
||||
}
|
||||
|
||||
// TCPConn 转换为TCPConn
|
||||
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
|
||||
conn, ok := this.rawConn.(*net.TCPConn)
|
||||
return conn, ok
|
||||
func (this *BaseClientConn) TCPConn() (tcpConn *net.TCPConn, ok bool) {
|
||||
// 设置包装前连接
|
||||
switch conn := this.rawConn.(type) {
|
||||
case *tls.Conn:
|
||||
var internalConn = conn.NetConn()
|
||||
clientConn, ok := internalConn.(*ClientConn)
|
||||
if ok {
|
||||
return clientConn.TCPConn()
|
||||
}
|
||||
tcpConn, ok = internalConn.(*net.TCPConn)
|
||||
default:
|
||||
tcpConn, ok = this.rawConn.(*net.TCPConn)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// SetLinger 设置Linger
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"net"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClientListener 客户端网络监听
|
||||
@@ -42,10 +43,28 @@ func (this *ClientListener) Accept() (net.Conn, error) {
|
||||
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
|
||||
if err == nil {
|
||||
canGoNext, _ := iplibrary.AllowIP(ip, 0)
|
||||
var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
|
||||
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
|
||||
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
|
||||
expiresAt, ok := waf.SharedIPBlackList.ContainsExpires(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
|
||||
if ok {
|
||||
var timeout = expiresAt - time.Now().Unix()
|
||||
if timeout > 0 {
|
||||
canGoNext = false
|
||||
|
||||
if !canGoNext || beingDenied {
|
||||
if timeout > 3600 {
|
||||
timeout = 3600
|
||||
}
|
||||
|
||||
// 使用本地防火墙延长封禁
|
||||
var fw = firewalls.Firewall()
|
||||
if fw != nil && !fw.IsMock() {
|
||||
// 这里 int(int64) 转换的前提是限制了 timeout <= 3600,否则将有整型溢出的风险
|
||||
_ = fw.DropSourceIP(ip, int(timeout), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !canGoNext {
|
||||
tcpConn, ok := conn.(*net.TCPConn)
|
||||
if ok {
|
||||
_ = tcpConn.SetLinger(0)
|
||||
@@ -53,14 +72,6 @@ func (this *ClientListener) Accept() (net.Conn, error) {
|
||||
|
||||
_ = conn.Close()
|
||||
|
||||
// 使用本地防火墙延长封禁
|
||||
if beingDenied {
|
||||
var fw = firewalls.Firewall()
|
||||
if fw != nil && !fw.IsMock() {
|
||||
_ = fw.DropSourceIP(ip, 60)
|
||||
}
|
||||
}
|
||||
|
||||
return this.Accept()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ Loop:
|
||||
this.rpcClient = client
|
||||
}
|
||||
|
||||
_, err := this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
_, err := this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
if err != nil {
|
||||
// 是否包含了invalid UTF-8
|
||||
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
|
||||
@@ -105,7 +105,7 @@ Loop:
|
||||
}
|
||||
|
||||
// 重新提交
|
||||
_, err = this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
|
||||
// logs.PrintAsJSON(accessLog)
|
||||
|
||||
//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
|
||||
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
accessLog,
|
||||
}})
|
||||
if err != nil {
|
||||
@@ -99,7 +99,7 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
accessLog,
|
||||
}})
|
||||
if err != nil {
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -82,7 +81,7 @@ func (this *HTTPCacheTaskManager) Start() {
|
||||
|
||||
if rpcClient != nil {
|
||||
for taskReq := range this.taskQueue {
|
||||
_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
|
||||
_, err := rpcClient.ServerRPC.PurgeServerCache(rpcClient.Context(), taskReq)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
|
||||
}
|
||||
@@ -105,8 +104,12 @@ func (this *HTTPCacheTaskManager) Loop() error {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
|
||||
resp, err := rpcClient.HTTPCacheTaskKeyRPC.FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
|
||||
if err != nil {
|
||||
// 忽略连接错误
|
||||
if rpc.IsConnError(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -132,7 +135,7 @@ func (this *HTTPCacheTaskManager) Loop() error {
|
||||
pbResults = append(pbResults, pbResult)
|
||||
}
|
||||
|
||||
_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
|
||||
_, err = rpcClient.HTTPCacheTaskKeyRPC.UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -236,7 +239,7 @@ func (this *HTTPCacheTaskManager) fetchKey(key *pb.HTTPCacheTaskKey) error {
|
||||
}()
|
||||
|
||||
// 读取内容,以便于生成缓存
|
||||
_, _ = io.Copy(ioutil.Discard, resp.Body)
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
|
||||
// 处理502
|
||||
if resp.StatusCode == http.StatusBadGateway {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
|
||||
}
|
||||
|
||||
var key = origin.UniqueKey() + "@" + originAddr
|
||||
var isLnRequest = origin.Id == 0
|
||||
|
||||
this.locker.RLock()
|
||||
client, found := this.clientsMap[key]
|
||||
@@ -101,6 +102,17 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
|
||||
idleConns = numberCPU * 8
|
||||
}
|
||||
|
||||
// 可以判断为Ln节点请求
|
||||
if isLnRequest {
|
||||
maxConnections *= 8
|
||||
idleConns *= 8
|
||||
idleTimeout *= 4
|
||||
} else if sharedNodeConfig != nil && sharedNodeConfig.Level > 1 {
|
||||
// Ln节点可以适当增加连接数
|
||||
maxConnections *= 2
|
||||
idleConns *= 2
|
||||
}
|
||||
|
||||
// TLS通讯
|
||||
var tlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
@@ -149,6 +161,7 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
TLSClientConfig: tlsConfig,
|
||||
ReadBufferSize: 8 * 1024,
|
||||
Proxy: nil,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestHTTPClientPool_Client(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHTTPClientPool_cleanClients(t *testing.T) {
|
||||
origin := &serverconfigs.OriginConfig{
|
||||
var origin = &serverconfigs.OriginConfig{
|
||||
Id: 1,
|
||||
Version: 2,
|
||||
Addr: &serverconfigs.NetworkAddressConfig{Host: "127.0.0.1", PortRange: "1234"},
|
||||
@@ -48,8 +48,7 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pool := NewHTTPClientPool()
|
||||
pool.clientExpiredDuration = 2 * time.Second
|
||||
var pool = NewHTTPClientPool()
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
t.Log("get", i)
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/metrics"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -151,7 +150,7 @@ func (this *HTTPRequest) Do() {
|
||||
// Web配置
|
||||
err := this.configureWeb(this.ReqServer.Web, true, 0)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to configure the server", "配置服务失败", false)
|
||||
this.doEnd()
|
||||
return
|
||||
}
|
||||
@@ -284,12 +283,12 @@ func (this *HTTPRequest) doBegin() {
|
||||
this.web.AccessLogRef.IsOn &&
|
||||
this.web.AccessLogRef.ContainsField(serverconfigs.HTTPAccessLogFieldRequestBody) {
|
||||
var err error
|
||||
this.requestBodyData, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
|
||||
this.requestBodyData, err = io.ReadAll(io.LimitReader(this.RawReq.Body, AccessLogMaxRequestBodySize))
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusBadGateway, false)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to read request body for access log", "为访问日志读取请求Body失败", false)
|
||||
return
|
||||
}
|
||||
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
|
||||
this.RawReq.Body = io.NopCloser(io.MultiReader(bytes.NewBuffer(this.requestBodyData), this.RawReq.Body))
|
||||
}
|
||||
|
||||
// 跳转
|
||||
@@ -678,7 +677,7 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
case "remoteAddrValue":
|
||||
return this.requestRemoteAddr(false)
|
||||
case "rawRemoteAddr":
|
||||
addr := this.RawReq.RemoteAddr
|
||||
var addr = this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err == nil {
|
||||
addr = host
|
||||
@@ -929,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
|
||||
// geo
|
||||
if prefix == "geo" {
|
||||
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
|
||||
var result = iplib.LookupIP(this.requestRemoteAddr(true))
|
||||
|
||||
switch suffix {
|
||||
case "country.name":
|
||||
if result != nil {
|
||||
return result.Country
|
||||
if result != nil && result.IsOk() {
|
||||
return result.CountryName()
|
||||
}
|
||||
return ""
|
||||
case "country.id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.CountryId())
|
||||
}
|
||||
return "0"
|
||||
case "province.name":
|
||||
if result != nil {
|
||||
return result.Province
|
||||
if result != nil && result.IsOk() {
|
||||
return result.ProvinceName()
|
||||
}
|
||||
return ""
|
||||
case "province.id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.ProvinceId())
|
||||
}
|
||||
return "0"
|
||||
case "city.name":
|
||||
if result != nil {
|
||||
return result.City
|
||||
if result != nil && result.IsOk() {
|
||||
return result.CityName()
|
||||
}
|
||||
return ""
|
||||
case "city.id":
|
||||
if result != nil {
|
||||
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
|
||||
if provinceId > 0 {
|
||||
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
|
||||
} else {
|
||||
return "0"
|
||||
}
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.CityId())
|
||||
}
|
||||
return "0"
|
||||
case "town.name":
|
||||
if result != nil && result.IsOk() {
|
||||
return result.TownName()
|
||||
}
|
||||
return ""
|
||||
case "town.id":
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.TownId())
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
@@ -972,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
|
||||
// ips
|
||||
if prefix == "isp" {
|
||||
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
|
||||
var result = iplib.LookupIP(this.requestRemoteAddr(true))
|
||||
|
||||
switch suffix {
|
||||
case "name":
|
||||
if result != nil {
|
||||
return result.ISP
|
||||
if result != nil && result.IsOk() {
|
||||
return result.ProviderName()
|
||||
}
|
||||
case "id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.ProviderId())
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
@@ -1099,7 +1103,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
}
|
||||
|
||||
// Remote-Addr
|
||||
remoteAddr := this.RawReq.RemoteAddr
|
||||
var remoteAddr = this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err == nil {
|
||||
if supportVar {
|
||||
@@ -1168,7 +1172,7 @@ func (this *HTTPRequest) requestRemoteUser() string {
|
||||
|
||||
// Path 请求的URL中路径部分
|
||||
func (this *HTTPRequest) Path() string {
|
||||
uri, err := url.ParseRequestURI(this.rawURI)
|
||||
uri, err := url.ParseRequestURI(this.uri)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -1316,7 +1320,7 @@ func (this *HTTPRequest) RemoteAddr() string {
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) RawRemoteAddr() string {
|
||||
addr := this.RawReq.RemoteAddr
|
||||
var addr = this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err == nil {
|
||||
addr = host
|
||||
@@ -1390,12 +1394,12 @@ func (this *HTTPRequest) Cookie(name string) string {
|
||||
return c.Value
|
||||
}
|
||||
|
||||
// DeleteHeader 删除Header
|
||||
// DeleteHeader 删除请求Header
|
||||
func (this *HTTPRequest) DeleteHeader(name string) {
|
||||
this.RawReq.Header.Del(name)
|
||||
}
|
||||
|
||||
// SetHeader 设置Header
|
||||
// SetHeader 设置请求Header
|
||||
func (this *HTTPRequest) SetHeader(name string, values []string) {
|
||||
this.RawReq.Header[name] = values
|
||||
}
|
||||
@@ -1424,11 +1428,16 @@ func (this *HTTPRequest) Done() {
|
||||
func (this *HTTPRequest) Close() {
|
||||
this.Done()
|
||||
|
||||
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if requestConn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
lingerConn, ok := requestConn.(LingerConn)
|
||||
if ok {
|
||||
_ = lingerConn.SetLinger(0)
|
||||
}
|
||||
|
||||
conn, ok := requestConn.(net.Conn)
|
||||
if ok {
|
||||
_ = conn.Close()
|
||||
@@ -1712,12 +1721,17 @@ func (this *HTTPRequest) canIgnore(err error) bool {
|
||||
}
|
||||
|
||||
// 客户端主动取消
|
||||
if err == errWritingToClient || err == context.Canceled || err == io.ErrShortWrite || strings.Contains(err.Error(), "write: connection timed out") || strings.Contains(err.Error(), "write: broken pipe") {
|
||||
if err == errWritingToClient ||
|
||||
err == context.Canceled ||
|
||||
err == io.ErrShortWrite ||
|
||||
strings.Contains(err.Error(), "write: connection") ||
|
||||
strings.Contains(err.Error(), "write: broken pipe") ||
|
||||
strings.Contains(err.Error(), "write tcp") {
|
||||
return true
|
||||
}
|
||||
|
||||
// HTTP/2流错误
|
||||
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
|
||||
if err.Error() == "http2: stream closed" || strings.Contains(err.Error(), "stream error") || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ func (this *HTTPRequest) doACME() (shouldStop bool) {
|
||||
return false
|
||||
}
|
||||
|
||||
keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
|
||||
keyResp, err := rpcClient.ACMEAuthenticationRPC.FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
|
||||
if err != nil {
|
||||
remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
|
||||
return false
|
||||
|
||||
@@ -5,7 +5,7 @@ package nodes
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
@@ -19,41 +19,52 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
|
||||
continue
|
||||
}
|
||||
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
|
||||
if !ref.AuthPolicy.MatchRequest(this.RawReq) {
|
||||
continue
|
||||
}
|
||||
ok, newURI, uriChanged, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
|
||||
subReq.TLS = this.RawReq.TLS
|
||||
subReq.RemoteAddr = this.RawReq.RemoteAddr
|
||||
subReq.Host = this.RawReq.Host
|
||||
subReq.Proto = this.RawReq.Proto
|
||||
subReq.ProtoMinor = this.RawReq.ProtoMinor
|
||||
subReq.ProtoMajor = this.RawReq.ProtoMajor
|
||||
subReq.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
|
||||
subReq.Body = io.NopCloser(bytes.NewReader([]byte{}))
|
||||
subReq.Header.Set("Referer", this.URL())
|
||||
var writer = NewEmptyResponseWriter(this.writer)
|
||||
this.doSubRequest(writer, subReq)
|
||||
return writer.StatusCode(), nil
|
||||
}, this.Format)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
|
||||
return
|
||||
}
|
||||
if b {
|
||||
if ok {
|
||||
if uriChanged {
|
||||
this.uri = newURI
|
||||
}
|
||||
this.tags = append(this.tags, ref.AuthPolicy.Type)
|
||||
return
|
||||
} else {
|
||||
// Basic Auth比较特殊
|
||||
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
|
||||
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.ReqHost
|
||||
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
if ok {
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.ReqHost
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
this.writer.WriteHeader(http.StatusUnauthorized)
|
||||
this.tags = append(this.tags, ref.AuthPolicy.Type)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,12 +44,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
// 检查服务独立的缓存条件
|
||||
refType := ""
|
||||
for _, cacheRef := range this.web.Cache.CacheRefs {
|
||||
if !cacheRef.IsOn ||
|
||||
cacheRef.Conds == nil ||
|
||||
!cacheRef.Conds.HasRequestConds() {
|
||||
if !cacheRef.IsOn {
|
||||
continue
|
||||
}
|
||||
if cacheRef.Conds.MatchRequest(this.Format) {
|
||||
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
|
||||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
|
||||
if cacheRef.IsReverse {
|
||||
return
|
||||
}
|
||||
@@ -61,12 +60,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
|
||||
// 检查策略默认的缓存条件
|
||||
for _, cacheRef := range cachePolicy.CacheRefs {
|
||||
if !cacheRef.IsOn ||
|
||||
cacheRef.Conds == nil ||
|
||||
!cacheRef.Conds.HasRequestConds() {
|
||||
if !cacheRef.IsOn {
|
||||
continue
|
||||
}
|
||||
if cacheRef.Conds.MatchRequest(this.Format) {
|
||||
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
|
||||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
|
||||
if cacheRef.IsReverse {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,32 +1,59 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const httpStatusPageTemplate = `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>${status} ${statusMessage}</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<h1>${status} ${statusMessage}</h1>
|
||||
<p>${message}</p>
|
||||
|
||||
<address>Request ID: ${requestId}.</address>
|
||||
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
func (this *HTTPRequest) write404() {
|
||||
if this.doPage(http.StatusNotFound) {
|
||||
this.writeCode(http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) writeCode(statusCode int) {
|
||||
if this.doPage(statusCode) {
|
||||
return
|
||||
}
|
||||
|
||||
this.processResponseHeaders(http.StatusNotFound)
|
||||
this.writer.WriteHeader(http.StatusNotFound)
|
||||
_, _ = this.writer.Write([]byte("404 page not found: '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
|
||||
var pageContent = configutils.ParseVariables(httpStatusPageTemplate, func(varName string) (value string) {
|
||||
switch varName {
|
||||
case "status":
|
||||
return types.String(statusCode)
|
||||
case "statusMessage":
|
||||
return http.StatusText(statusCode)
|
||||
case "requestId":
|
||||
return this.requestId
|
||||
case "message":
|
||||
return "" // 空
|
||||
}
|
||||
return "${" + varName + "}"
|
||||
})
|
||||
|
||||
this.processResponseHeaders(statusCode)
|
||||
this.writer.WriteHeader(statusCode)
|
||||
|
||||
_, _ = this.writer.Write([]byte(pageContent))
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) writeCode(code int) {
|
||||
if this.doPage(code) {
|
||||
return
|
||||
}
|
||||
|
||||
this.processResponseHeaders(code)
|
||||
this.writer.WriteHeader(code)
|
||||
_, _ = this.writer.Write([]byte(types.String(code) + " " + http.StatusText(code) + ": '" + this.URL() + "'" + " (Request Id: " + this.requestId + ")"))
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
|
||||
func (this *HTTPRequest) write50x(err error, statusCode int, enMessage string, zhMessage string, canTryStale bool) {
|
||||
if err != nil {
|
||||
this.addError(err)
|
||||
}
|
||||
@@ -37,7 +64,7 @@ func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
|
||||
this.web.Cache.Stale != nil &&
|
||||
this.web.Cache.Stale.IsOn &&
|
||||
(len(this.web.Cache.Stale.Status) == 0 || lists.ContainsInt(this.web.Cache.Stale.Status, statusCode)) {
|
||||
ok := this.doCacheRead(true)
|
||||
var ok = this.doCacheRead(true)
|
||||
if ok {
|
||||
return
|
||||
}
|
||||
@@ -47,7 +74,34 @@ func (this *HTTPRequest) write50x(err error, statusCode int, canTryStale bool) {
|
||||
if this.doPage(statusCode) {
|
||||
return
|
||||
}
|
||||
|
||||
// 内置HTML模板
|
||||
var pageContent = configutils.ParseVariables(httpStatusPageTemplate, func(varName string) (value string) {
|
||||
switch varName {
|
||||
case "status":
|
||||
return types.String(statusCode)
|
||||
case "statusMessage":
|
||||
return http.StatusText(statusCode)
|
||||
case "requestId":
|
||||
return this.requestId
|
||||
case "message":
|
||||
var acceptLanguages = this.RawReq.Header.Get("Accept-Language")
|
||||
if len(acceptLanguages) > 0 {
|
||||
var index = strings.Index(acceptLanguages, ",")
|
||||
if index > 0 {
|
||||
var firstLanguage = acceptLanguages[:index]
|
||||
if firstLanguage == "zh-CN" {
|
||||
return "网站出了一点小问题,原因:" + zhMessage + "。"
|
||||
}
|
||||
}
|
||||
}
|
||||
return "The site is unavailable now, cause: " + enMessage + "."
|
||||
}
|
||||
return "${" + varName + "}"
|
||||
})
|
||||
|
||||
this.processResponseHeaders(statusCode)
|
||||
this.writer.WriteHeader(statusCode)
|
||||
_, _ = this.writer.Write([]byte(types.String(statusCode) + " " + http.StatusText(statusCode) + " (Request Id: " + this.requestId + ")"))
|
||||
|
||||
_, _ = this.writer.Write([]byte(pageContent))
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
|
||||
client, err := fcgi.SharedPool(fastcgi.Network(), fastcgi.RealAddress(), uint(poolSize)).Client()
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to create Fastcgi pool", "Fastcgi池生成失败", false)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -159,13 +159,13 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
|
||||
|
||||
resp, stderr, err := client.Call(fcgiReq)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to read Fastcgi", "读取Fastcgi失败", false)
|
||||
return
|
||||
}
|
||||
|
||||
if len(stderr) > 0 {
|
||||
err := errors.New("Fastcgi Error: " + strings.TrimSpace(string(stderr)) + " script: " + maps.NewMap(params).GetString("SCRIPT_FILENAME"))
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to read Fastcgi", "读取Fastcgi失败", false)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,6 @@ func (this *HTTPRequest) checkLnRequest() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) getLnOrigin() *serverconfigs.OriginConfig {
|
||||
return nil
|
||||
func (this *HTTPRequest) getLnOrigin(excludingNodeIds []int64) (originConfig *serverconfigs.OriginConfig, lnNodeId int64, hasMultipleNodes bool) {
|
||||
return nil, 0, false
|
||||
}
|
||||
|
||||
@@ -34,17 +34,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
|
||||
}
|
||||
return this.RawReq.ContentLength + hl, true
|
||||
case "${countConnection}":
|
||||
metricNewConnMapLocker.Lock()
|
||||
_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
|
||||
if ok {
|
||||
delete(metricNewConnMap, this.RawReq.RemoteAddr)
|
||||
}
|
||||
metricNewConnMapLocker.Unlock()
|
||||
if ok {
|
||||
return 1, true
|
||||
} else {
|
||||
return 0, false
|
||||
}
|
||||
return 1, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
@@ -21,10 +21,32 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
return
|
||||
}
|
||||
|
||||
var retries = 3
|
||||
|
||||
var failedOriginIds []int64
|
||||
var failedLnNodeIds []int64
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
originId, lnNodeId, shouldRetry := this.doOriginRequest(failedOriginIds, failedLnNodeIds, i == 0, i == retries-1)
|
||||
if !shouldRetry {
|
||||
break
|
||||
}
|
||||
if originId > 0 {
|
||||
failedOriginIds = append(failedOriginIds, originId)
|
||||
}
|
||||
if lnNodeId > 0 {
|
||||
failedLnNodeIds = append(failedLnNodeIds, lnNodeId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 请求源站
|
||||
func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeIds []int64, isFirstTry bool, isLastRetry bool) (originId int64, lnNodeId int64, shouldRetry bool) {
|
||||
// 对URL的处理
|
||||
var stripPrefix = this.reverseProxy.StripPrefix
|
||||
var requestURI = this.reverseProxy.RequestURI
|
||||
var requestURIHasVariables = this.reverseProxy.RequestURIHasVariables()
|
||||
var oldURI = this.uri
|
||||
|
||||
var requestHost = ""
|
||||
if this.reverseProxy.RequestHostType == serverconfigs.RequestHostTypeCustomized {
|
||||
@@ -41,24 +63,39 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
var origin *serverconfigs.OriginConfig
|
||||
|
||||
// 二级节点
|
||||
var hasMultipleLnNodes = false
|
||||
if this.cacheRef != nil {
|
||||
origin = this.getLnOrigin()
|
||||
origin, lnNodeId, hasMultipleLnNodes = this.getLnOrigin(failedLnNodeIds)
|
||||
if origin != nil {
|
||||
// 强制变更原来访问的域名
|
||||
requestHost = this.ReqHost
|
||||
}
|
||||
|
||||
// 回源Header中去除If-None-Match和If-Modified-Since
|
||||
if !this.cacheRef.EnableIfNoneMatch {
|
||||
this.DeleteHeader("If-None-Match")
|
||||
}
|
||||
if !this.cacheRef.EnableIfModifiedSince {
|
||||
this.DeleteHeader("If-Modified-Since")
|
||||
}
|
||||
}
|
||||
|
||||
// 自定义源站
|
||||
if origin == nil {
|
||||
origin = this.reverseProxy.NextOrigin(requestCall)
|
||||
if !isFirstTry {
|
||||
origin = this.reverseProxy.AnyOrigin(requestCall, failedOriginIds)
|
||||
}
|
||||
if origin == nil {
|
||||
origin = this.reverseProxy.NextOrigin(requestCall)
|
||||
}
|
||||
requestCall.CallResponseCallbacks(this.writer)
|
||||
if origin == nil {
|
||||
err := errors.New(this.URL() + ": no available origin sites for reverse proxy")
|
||||
remotelogs.ServerError(this.ReqServer.Id, "HTTP_REQUEST_REVERSE_PROXY", err.Error(), "", nil)
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
this.write50x(err, http.StatusBadGateway, "No origin site yet", "尚未配置源站", true)
|
||||
return
|
||||
}
|
||||
originId = origin.Id
|
||||
|
||||
if len(origin.StripPrefix) > 0 {
|
||||
stripPrefix = origin.StripPrefix
|
||||
@@ -80,7 +117,7 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
if origin.Addr == nil {
|
||||
err := errors.New(this.URL() + ": Origin '" + strconv.FormatInt(origin.Id, 10) + "' does not has a address")
|
||||
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
this.write50x(err, http.StatusBadGateway, "Origin site did not has a valid address", "源站尚未配置地址", true)
|
||||
return
|
||||
}
|
||||
this.RawReq.URL.Scheme = origin.Addr.Protocol.Primary().Scheme()
|
||||
@@ -132,7 +169,7 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
if originHostIndex < 0 {
|
||||
var originErr = errors.New(this.URL() + ": Invalid origin address '" + originAddr + "', lacking port")
|
||||
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", originErr.Error())
|
||||
this.write50x(originErr, http.StatusBadGateway, true)
|
||||
this.write50x(originErr, http.StatusBadGateway, "No port in origin site address", "源站地址中没有配置端口", true)
|
||||
return
|
||||
}
|
||||
originAddr = originAddr[:originHostIndex+1] + types.String(this.requestServerPort())
|
||||
@@ -211,16 +248,10 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol, this.reverseProxy.FollowRedirects)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Create client failed: "+err.Error())
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to create origin site client", "构造源站客户端失败", true)
|
||||
return
|
||||
}
|
||||
|
||||
// 在HTTP/2下需要防止因为requestBody而导致Content-Length为空的问题
|
||||
if this.RawReq.ProtoMajor == 2 && this.RawReq.ContentLength == 0 {
|
||||
_ = this.RawReq.Body.Close()
|
||||
this.RawReq.Body = nil
|
||||
}
|
||||
|
||||
// 开始请求
|
||||
resp, err := client.Do(this.RawReq)
|
||||
if err != nil {
|
||||
@@ -230,18 +261,35 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
|
||||
this.reverseProxy.ResetScheduling()
|
||||
})
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
|
||||
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.RawReq.URL.String()+": Request origin server failed: "+err.Error())
|
||||
} else if httpErr.Err != context.Canceled {
|
||||
SharedOriginStateManager.Fail(origin, requestHost, this.reverseProxy, func() {
|
||||
this.reverseProxy.ResetScheduling()
|
||||
})
|
||||
|
||||
// 是否需要重试
|
||||
if (originId > 0 || (lnNodeId > 0 && hasMultipleLnNodes)) && !isLastRetry {
|
||||
shouldRetry = true
|
||||
this.uri = oldURI // 恢复备份
|
||||
|
||||
if resp != nil && resp.Body != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
if httpErr.Err != io.EOF {
|
||||
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if httpErr.Timeout() {
|
||||
this.write50x(err, http.StatusGatewayTimeout, true)
|
||||
this.write50x(err, http.StatusGatewayTimeout, "Read origin site timeout", "源站读取超时", true)
|
||||
} else if httpErr.Temporary() {
|
||||
this.write50x(err, http.StatusServiceUnavailable, true)
|
||||
this.write50x(err, http.StatusServiceUnavailable, "Origin site unavailable now", "源站当前不可用", true)
|
||||
} else {
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
|
||||
}
|
||||
if httpErr.Err != io.EOF {
|
||||
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", this.URL()+": Request origin server failed: "+err.Error())
|
||||
@@ -264,7 +312,7 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
}
|
||||
|
||||
if !isClientError {
|
||||
this.write50x(err, http.StatusBadGateway, true)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to read origin site", "源站读取失败", true)
|
||||
}
|
||||
}
|
||||
if resp != nil && resp.Body != nil {
|
||||
@@ -410,4 +458,6 @@ func (this *HTTPRequest) doReverseProxy() {
|
||||
if (err == nil || err == io.EOF) && (closeErr == nil || closeErr == io.EOF) {
|
||||
this.writer.SetOk()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -114,7 +114,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
}
|
||||
return
|
||||
} else {
|
||||
this.write50x(err, http.StatusInternalServerError, true)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to stat the file", "查看文件统计信息失败", true)
|
||||
if !this.canIgnore(err) {
|
||||
logs.Error(err)
|
||||
}
|
||||
@@ -145,7 +145,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
}
|
||||
return
|
||||
} else {
|
||||
this.write50x(err, http.StatusInternalServerError, true)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to stat the file", "查看文件统计信息失败", true)
|
||||
if !this.canIgnore(err) {
|
||||
logs.Error(err)
|
||||
}
|
||||
@@ -285,7 +285,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
|
||||
|
||||
fileReader, err := os.OpenFile(filePath, os.O_RDONLY, 0444)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, true)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to open the file", "试图打开文件失败", true)
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ func (this *HTTPRequest) doURL(method string, url string, host string, statusCod
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_URL", req.URL.String()+": "+err.Error())
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to read url", "读取URL失败", false)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
|
||||
@@ -180,14 +180,14 @@ var httpRequestTimestamp int64
|
||||
var httpRequestId int32 = 1_000_000
|
||||
|
||||
func httpRequestNextId() string {
|
||||
var unixTime = utils.UnixTimeMilli()
|
||||
unixTime, unixTimeString := utils.UnixTimeMilliString()
|
||||
if unixTime > httpRequestTimestamp {
|
||||
atomic.StoreInt32(&httpRequestId, 1_000_000)
|
||||
httpRequestTimestamp = unixTime
|
||||
}
|
||||
|
||||
// timestamp + requestId + nodeId
|
||||
return strconv.FormatInt(unixTime, 10) + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
|
||||
// timestamp + nodeId + requestId
|
||||
return unixTimeString + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
|
||||
}
|
||||
|
||||
// 检查是否可以接受某个编码
|
||||
|
||||
@@ -2,6 +2,7 @@ package nodes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
@@ -11,7 +12,6 @@ import (
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
@@ -162,56 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
|
||||
// 检查地区封禁
|
||||
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
|
||||
if iplibrary.SharedLibrary != nil {
|
||||
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
|
||||
regionConfig := firewallPolicy.Inbound.Region
|
||||
if regionConfig.IsNotEmpty() {
|
||||
for _, remoteAddr := range remoteAddrs {
|
||||
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
|
||||
} else if result != nil {
|
||||
// 检查国家级别封禁
|
||||
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
|
||||
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
|
||||
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
|
||||
regionConfig := firewallPolicy.Inbound.Region
|
||||
if regionConfig.IsNotEmpty() {
|
||||
for _, remoteAddr := range remoteAddrs {
|
||||
var result = iplib.LookupIP(remoteAddr)
|
||||
if result != nil && result.IsOk() {
|
||||
// 检查国家/地区级别封禁
|
||||
var countryId = result.CountryId()
|
||||
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyCountry")
|
||||
}
|
||||
|
||||
return true, false
|
||||
}
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyCountry")
|
||||
}
|
||||
|
||||
// 检查省份封禁
|
||||
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
|
||||
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
|
||||
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
return true, false
|
||||
}
|
||||
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
// 检查省份封禁
|
||||
var provinceId = result.ProvinceId()
|
||||
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyProvince")
|
||||
}
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
|
||||
return true, false
|
||||
}
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyProvince")
|
||||
}
|
||||
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -359,7 +351,7 @@ func (this *HTTPRequest) WAFSetCacheBody(body []byte) {
|
||||
// WAFReadBody 读取Body
|
||||
func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
|
||||
if this.RawReq.ContentLength > 0 {
|
||||
data, err = ioutil.ReadAll(io.LimitReader(this.RawReq.Body, max))
|
||||
data, err = io.ReadAll(io.LimitReader(this.RawReq.Body, max))
|
||||
}
|
||||
|
||||
return
|
||||
@@ -368,7 +360,7 @@ func (this *HTTPRequest) WAFReadBody(max int64) (data []byte, err error) {
|
||||
// WAFRestoreBody 恢复Body
|
||||
func (this *HTTPRequest) WAFRestoreBody(data []byte) {
|
||||
if len(data) > 0 {
|
||||
this.RawReq.Body = ioutil.NopCloser(io.MultiReader(bytes.NewBuffer(data), this.RawReq.Body))
|
||||
this.RawReq.Body = io.NopCloser(io.MultiReader(bytes.NewBuffer(data), this.RawReq.Body))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -380,6 +372,8 @@ func (this *HTTPRequest) WAFServerId() int64 {
|
||||
// WAFClose 关闭连接
|
||||
func (this *HTTPRequest) WAFClose() {
|
||||
this.Close()
|
||||
|
||||
// 这里不要强关IP所有连接,避免因为单个服务而影响所有
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {
|
||||
|
||||
@@ -43,7 +43,7 @@ func (this *HTTPRequest) doWebsocket(requestHost string) {
|
||||
// TODO 增加N次错误重试,重试的时候需要尝试不同的源站
|
||||
originConn, _, err := OriginConnect(this.origin, this.requestServerPort(), this.RawReq.RemoteAddr, requestHost)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusBadGateway, false)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to connect origin site", "源站连接失败", false)
|
||||
|
||||
// 增加失败次数
|
||||
SharedOriginStateManager.Fail(this.origin, requestHost, this.reverseProxy, func() {
|
||||
@@ -65,13 +65,13 @@ func (this *HTTPRequest) doWebsocket(requestHost string) {
|
||||
|
||||
err = this.RawReq.Write(originConn)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusBadGateway, false)
|
||||
this.write50x(err, http.StatusBadGateway, "Failed to write request to origin site", "源站请求初始化失败", false)
|
||||
return
|
||||
}
|
||||
|
||||
clientConn, _, err := this.writer.Hijack()
|
||||
if err != nil || clientConn == nil {
|
||||
this.write50x(err, http.StatusInternalServerError, false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to get origin site connection", "获取源站连接失败", false)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
|
||||
@@ -26,7 +26,6 @@ import (
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
@@ -524,7 +523,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
|
||||
|
||||
this.webpOriginContentType = contentType
|
||||
this.webpIsEncoding = true
|
||||
resp.Body = ioutil.NopCloser(&bytes.Buffer{})
|
||||
resp.Body = io.NopCloser(&bytes.Buffer{})
|
||||
this.delayRead = true
|
||||
|
||||
this.Header().Del("Content-Length")
|
||||
@@ -1008,30 +1007,35 @@ func (this *HTTPWriter) finishWebP() {
|
||||
Exact: true,
|
||||
})
|
||||
} else if gifImage != nil {
|
||||
anim := gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||
|
||||
anim.WebPAnimEncoderOptions.SetKmin(9)
|
||||
anim.WebPAnimEncoderOptions.SetKmax(17)
|
||||
defer anim.ReleaseMemory()
|
||||
webpConfig := gowebp.NewWebpConfig()
|
||||
var webpConfig = gowebp.NewWebpConfig()
|
||||
//webpConfig.SetLossless(1)
|
||||
webpConfig.SetQuality(f)
|
||||
|
||||
var timeline = 0
|
||||
|
||||
var lastErr error
|
||||
for i, img := range gifImage.Image {
|
||||
err = anim.AddFrame(img, timeline, webpConfig)
|
||||
if err != nil {
|
||||
break
|
||||
// 有错误直接跳过
|
||||
lastErr = err
|
||||
err = nil
|
||||
}
|
||||
timeline += gifImage.Delay[i] * 10
|
||||
}
|
||||
if err == nil {
|
||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||
|
||||
if err == nil {
|
||||
err = anim.Encode(this.writer)
|
||||
}
|
||||
if lastErr != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
|
||||
}
|
||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||
|
||||
if err == nil {
|
||||
err = anim.Encode(this.writer)
|
||||
}
|
||||
|
||||
anim.ReleaseMemory()
|
||||
}
|
||||
|
||||
if err != nil && !this.req.canIgnore(err) {
|
||||
|
||||
103
internal/nodes/ip_library_updater.go
Normal file
103
internal/nodes/ip_library_updater.go
Normal file
@@ -0,0 +1,103 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type IPLibraryUpdater struct {
|
||||
}
|
||||
|
||||
func NewIPLibraryUpdater() *IPLibraryUpdater {
|
||||
return &IPLibraryUpdater{}
|
||||
}
|
||||
|
||||
// DataDir 文件目录
|
||||
func (this *IPLibraryUpdater) DataDir() string {
|
||||
// data/
|
||||
var dir = Tea.Root + "/data"
|
||||
stat, err := os.Stat(dir)
|
||||
if err == nil && stat.IsDir() {
|
||||
return dir
|
||||
}
|
||||
|
||||
err = os.Mkdir(dir, 0666)
|
||||
if err == nil {
|
||||
return dir
|
||||
}
|
||||
|
||||
remotelogs.Error("IP_LIBRARY_UPDATER", "create directory '"+dir+"' failed: "+err.Error())
|
||||
|
||||
// 如果不能创建 data/ 目录,那么使用临时目录
|
||||
return os.TempDir()
|
||||
}
|
||||
|
||||
// FindLatestFile 检查最新的IP库文件
|
||||
func (this *IPLibraryUpdater) FindLatestFile() (code string, fileId int64, err error) {
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
resp, err := rpcClient.IPLibraryArtifactRPC.FindPublicIPLibraryArtifact(rpcClient.Context(), &pb.FindPublicIPLibraryArtifactRequest{})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var artifact = resp.IpLibraryArtifact
|
||||
if artifact == nil {
|
||||
return
|
||||
}
|
||||
return artifact.Code, artifact.FileId, nil
|
||||
}
|
||||
|
||||
// DownloadFile 下载文件
|
||||
func (this *IPLibraryUpdater) DownloadFile(fileId int64, writer io.Writer) error {
|
||||
if fileId <= 0 {
|
||||
return errors.New("invalid fileId: " + types.String(fileId))
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunkIdsResp, err := rpcClient.FileChunkRPC.FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: fileId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, chunkId := range chunkIdsResp.FileChunkIds {
|
||||
chunkResp, err := rpcClient.FileChunkRPC.DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var chunk = chunkResp.FileChunk
|
||||
if chunk == nil {
|
||||
return errors.New("can not find file chunk with chunk id '" + types.String(chunkId) + "'")
|
||||
}
|
||||
_, err = writer.Write(chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LogInfo 普通日志
|
||||
func (this *IPLibraryUpdater) LogInfo(message string) {
|
||||
remotelogs.Println("IP_LIBRARY_UPDATER", message)
|
||||
}
|
||||
|
||||
// LogError 错误日志
|
||||
func (this *IPLibraryUpdater) LogError(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
remotelogs.Error("IP_LIBRARY_UPDATER", err.Error())
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/tls"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"golang.org/x/net/http2"
|
||||
"io"
|
||||
@@ -13,14 +12,11 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var httpErrorLogger = log.New(io.Discard, "", 0)
|
||||
var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool
|
||||
var metricNewConnMapLocker = &sync.Mutex{}
|
||||
|
||||
type contextKey struct {
|
||||
key string
|
||||
@@ -55,23 +51,10 @@ func (this *HTTPListener) Serve() error {
|
||||
switch state {
|
||||
case http.StateNew:
|
||||
atomic.AddInt64(&this.countActiveConnections, 1)
|
||||
|
||||
// 为指标存储连接信息
|
||||
if sharedNodeConfig.HasHTTPConnectionMetrics() {
|
||||
metricNewConnMapLocker.Lock()
|
||||
metricNewConnMap[conn.RemoteAddr().String()] = zero.New()
|
||||
metricNewConnMapLocker.Unlock()
|
||||
}
|
||||
case http.StateActive, http.StateIdle, http.StateHijacked:
|
||||
// Nothing to do
|
||||
case http.StateClosed:
|
||||
atomic.AddInt64(&this.countActiveConnections, -1)
|
||||
|
||||
// 移除指标存储连接信息
|
||||
// 因为中途配置可能有改变,所以暂时不添加条件
|
||||
metricNewConnMapLocker.Lock()
|
||||
delete(metricNewConnMap, conn.RemoteAddr().String())
|
||||
metricNewConnMapLocker.Unlock()
|
||||
}
|
||||
},
|
||||
ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
@@ -108,8 +109,9 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
|
||||
// 记录域名排行
|
||||
tlsConn, ok := conn.(*tls.Conn)
|
||||
var recordStat = false
|
||||
var serverName = ""
|
||||
if ok {
|
||||
var serverName = tlsConn.ConnectionState().ServerName
|
||||
serverName = tlsConn.ConnectionState().ServerName
|
||||
if len(serverName) > 0 {
|
||||
// 统计
|
||||
stats.SharedTrafficStatManager.Add(server.Id, serverName, 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
@@ -122,8 +124,9 @@ func (this *TCPListener) handleConn(conn net.Conn) error {
|
||||
stats.SharedTrafficStatManager.Add(server.Id, "", 0, 0, 1, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
}
|
||||
|
||||
originConn, err := this.connectOrigin(server.Id, server.ReverseProxy, conn.RemoteAddr().String())
|
||||
originConn, err := this.connectOrigin(server.Id, serverName, server.ReverseProxy, conn.RemoteAddr().String())
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -218,37 +221,63 @@ func (this *TCPListener) Close() error {
|
||||
}
|
||||
|
||||
// 连接源站
|
||||
func (this *TCPListener) connectOrigin(serverId int64, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
|
||||
func (this *TCPListener) connectOrigin(serverId int64, requestHost string, reverseProxy *serverconfigs.ReverseProxyConfig, remoteAddr string) (conn net.Conn, err error) {
|
||||
if reverseProxy == nil {
|
||||
return nil, errors.New("no reverse proxy config")
|
||||
}
|
||||
|
||||
var requestCall = shared.NewRequestCall()
|
||||
requestCall.Domain = requestHost
|
||||
|
||||
var retries = 3
|
||||
var addr string
|
||||
|
||||
var failedOriginIds []int64
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
var origin = reverseProxy.NextOrigin(nil)
|
||||
var origin *serverconfigs.OriginConfig
|
||||
if len(failedOriginIds) > 0 {
|
||||
origin = reverseProxy.AnyOrigin(requestCall, failedOriginIds)
|
||||
}
|
||||
if origin == nil {
|
||||
origin = reverseProxy.NextOrigin(requestCall)
|
||||
}
|
||||
if origin == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// 回源主机名
|
||||
var requestHost = ""
|
||||
if len(reverseProxy.RequestHost) > 0 {
|
||||
requestHost = reverseProxy.RequestHost
|
||||
}
|
||||
if len(origin.RequestHost) > 0 {
|
||||
requestHost = origin.RequestHost
|
||||
} else if len(reverseProxy.RequestHost) > 0 {
|
||||
requestHost = reverseProxy.RequestHost
|
||||
}
|
||||
|
||||
conn, addr, err = OriginConnect(origin, this.port, remoteAddr, requestHost)
|
||||
if err != nil {
|
||||
failedOriginIds = append(failedOriginIds, origin.Id)
|
||||
|
||||
remotelogs.ServerError(serverId, "TCP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
|
||||
|
||||
SharedOriginStateManager.Fail(origin, requestHost, reverseProxy, func() {
|
||||
reverseProxy.ResetScheduling()
|
||||
})
|
||||
|
||||
continue
|
||||
} else {
|
||||
if !origin.IsOk {
|
||||
SharedOriginStateManager.Success(origin, func() {
|
||||
reverseProxy.ResetScheduling()
|
||||
})
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
|
||||
|
||||
if err == nil {
|
||||
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,10 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
UDPConnLifeSeconds = 30
|
||||
)
|
||||
|
||||
type UDPListener struct {
|
||||
BaseListener
|
||||
|
||||
@@ -135,16 +139,39 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
|
||||
|
||||
var retries = 3
|
||||
var addr string
|
||||
|
||||
var failedOriginIds []int64
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
var origin = reverseProxy.NextOrigin(nil)
|
||||
var origin *serverconfigs.OriginConfig
|
||||
if len(failedOriginIds) > 0 {
|
||||
origin = reverseProxy.AnyOrigin(nil, failedOriginIds)
|
||||
}
|
||||
if origin == nil {
|
||||
origin = reverseProxy.NextOrigin(nil)
|
||||
}
|
||||
if origin == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
conn, addr, err = OriginConnect(origin, this.port, remoteAddr.String(), "")
|
||||
if err != nil {
|
||||
failedOriginIds = append(failedOriginIds, origin.Id)
|
||||
|
||||
remotelogs.ServerError(serverId, "UDP_LISTENER", "unable to connect origin server: "+addr+": "+err.Error(), "", nil)
|
||||
|
||||
SharedOriginStateManager.Fail(origin, "", reverseProxy, func() {
|
||||
reverseProxy.ResetScheduling()
|
||||
})
|
||||
|
||||
continue
|
||||
} else {
|
||||
if !origin.IsOk {
|
||||
SharedOriginStateManager.Success(origin, func() {
|
||||
reverseProxy.ResetScheduling()
|
||||
})
|
||||
}
|
||||
|
||||
// PROXY Protocol
|
||||
if reverseProxy != nil &&
|
||||
reverseProxy.ProxyProtocol != nil &&
|
||||
@@ -171,14 +198,17 @@ func (this *UDPListener) connectOrigin(serverId int64, reverseProxy *serverconfi
|
||||
return
|
||||
}
|
||||
}
|
||||
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
|
||||
|
||||
if err == nil {
|
||||
err = errors.New("server '" + types.String(serverId) + "': no available origin server can be used")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 回收连接
|
||||
func (this *UDPListener) gcConns() {
|
||||
this.connLocker.Lock()
|
||||
closingConns := []*UDPConn{}
|
||||
var closingConns = []*UDPConn{}
|
||||
for addr, conn := range this.connMap {
|
||||
if !conn.IsOk() {
|
||||
closingConns = append(closingConns, conn)
|
||||
@@ -203,7 +233,7 @@ type UDPConn struct {
|
||||
}
|
||||
|
||||
func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *net.UDPConn, serverConn *net.UDPConn) *UDPConn {
|
||||
conn := &UDPConn{
|
||||
var conn = &UDPConn{
|
||||
addr: addr,
|
||||
proxyConn: proxyConn,
|
||||
serverConn: serverConn,
|
||||
@@ -217,7 +247,7 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *ne
|
||||
}
|
||||
|
||||
goman.New(func() {
|
||||
buffer := utils.BytePool4k.Get()
|
||||
var buffer = utils.BytePool4k.Get()
|
||||
defer func() {
|
||||
utils.BytePool4k.Put(buffer)
|
||||
}()
|
||||
@@ -232,9 +262,13 @@ func NewUDPConn(server *serverconfigs.ServerConfig, addr net.Addr, proxyConn *ne
|
||||
break
|
||||
}
|
||||
|
||||
// 记录流量
|
||||
// 记录流量和带宽
|
||||
if server != nil {
|
||||
// 流量
|
||||
stats.SharedTrafficStatManager.Add(server.Id, "", int64(n), 0, 0, 0, 0, 0, server.ShouldCheckTrafficLimit(), server.PlanId())
|
||||
|
||||
// 带宽
|
||||
stats.SharedBandwidthStatManager.Add(server.UserId, server.Id, int64(n))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
@@ -268,5 +302,5 @@ func (this *UDPConn) IsOk() bool {
|
||||
if !this.isOk {
|
||||
return false
|
||||
}
|
||||
return time.Now().Unix()-this.activatedAt < 30 // 如果超过 N 秒没有活动我们认为是超时
|
||||
return time.Now().Unix()-this.activatedAt < UDPConnLifeSeconds // 如果超过 N 秒没有活动我们认为是超时
|
||||
}
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
|
||||
@@ -21,6 +23,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
@@ -30,7 +33,6 @@ import (
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"github.com/iwind/gosock/pkg/gosock"
|
||||
"gopkg.in/yaml.v3"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -79,7 +81,7 @@ func (this *Node) Test() error {
|
||||
if err != nil {
|
||||
return errors.New("test rpc failed: " + err.Error())
|
||||
}
|
||||
_, err = rpcClient.APINodeRPC().FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
|
||||
_, err = rpcClient.APINodeRPC.FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
|
||||
if err != nil {
|
||||
return errors.New("test rpc failed: " + err.Error())
|
||||
}
|
||||
@@ -102,9 +104,6 @@ func (this *Node) Start() {
|
||||
// 监听signal
|
||||
this.listenSignals()
|
||||
|
||||
// 启动事件
|
||||
events.Notify(events.EventStart)
|
||||
|
||||
// 本地Sock
|
||||
err := this.listenSock()
|
||||
if err != nil {
|
||||
@@ -112,9 +111,19 @@ func (this *Node) Start() {
|
||||
return
|
||||
}
|
||||
|
||||
// 启动IP库
|
||||
remotelogs.Println("NODE", "initializing ip library ...")
|
||||
err = iplib.InitDefault()
|
||||
if err != nil {
|
||||
remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
|
||||
}
|
||||
|
||||
// 检查硬盘类型
|
||||
this.checkDisk()
|
||||
|
||||
// 启动事件
|
||||
events.Notify(events.EventStart)
|
||||
|
||||
// 读取API配置
|
||||
remotelogs.Println("NODE", "init config ...")
|
||||
err = this.syncConfig(0)
|
||||
@@ -147,11 +156,21 @@ func (this *Node) Start() {
|
||||
// 启动同步计时器
|
||||
this.startSyncTimer()
|
||||
|
||||
// 状态变更计时器
|
||||
// 更新IP库
|
||||
goman.New(func() {
|
||||
iplib.NewUpdater(NewIPLibraryUpdater(), 10*time.Minute).Start()
|
||||
})
|
||||
|
||||
// 监控节点运行状态
|
||||
goman.New(func() {
|
||||
NewNodeStatusExecutor().Listen()
|
||||
})
|
||||
|
||||
// 同步时间
|
||||
goman.New(func() {
|
||||
clock.Start()
|
||||
})
|
||||
|
||||
// 读取配置
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
@@ -207,8 +226,9 @@ func (this *Node) Start() {
|
||||
|
||||
// Daemon 实现守护进程
|
||||
func (this *Node) Daemon() {
|
||||
isDebug := lists.ContainsString(os.Args, "debug")
|
||||
isDebug = true
|
||||
teaconst.IsDaemon = true
|
||||
|
||||
var isDebug = lists.ContainsString(os.Args, "debug")
|
||||
for {
|
||||
conn, err := this.sock.Dial()
|
||||
if err != nil {
|
||||
@@ -227,13 +247,18 @@ func (this *Node) Daemon() {
|
||||
_ = os.Setenv("EdgeDaemon", "on")
|
||||
_ = os.Setenv("EdgeBackground", "on")
|
||||
|
||||
cmd := exec.Command(exe)
|
||||
var cmd = exec.Command(exe)
|
||||
var buf = &bytes.Buffer{}
|
||||
cmd.Stderr = buf
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
if isDebug {
|
||||
log.Println("[DAEMON]" + buf.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -289,17 +314,25 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
var nodeCtx = rpcClient.Context()
|
||||
tasksResp, err := rpcClient.NodeTaskRPC().FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
|
||||
tasksResp, err := rpcClient.NodeTaskRPC.FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
|
||||
if err != nil {
|
||||
if rpc.IsConnError(err) && !Tea.IsTesting() {
|
||||
return nil
|
||||
}
|
||||
return errors.New("read node tasks failed: " + err.Error())
|
||||
}
|
||||
for _, task := range tasksResp.NodeTasks {
|
||||
switch task.Type {
|
||||
case "ipItemChanged":
|
||||
iplibrary.IPListUpdateNotify <- true
|
||||
// 防止阻塞
|
||||
select {
|
||||
case iplibrary.IPListUpdateNotify <- true:
|
||||
default:
|
||||
|
||||
}
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -318,13 +351,13 @@ func (this *Node) loop() error {
|
||||
err = this.syncConfig(task.Version)
|
||||
}
|
||||
if err != nil {
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: false,
|
||||
Error: err.Error(),
|
||||
})
|
||||
} else {
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -333,11 +366,12 @@ func (this *Node) loop() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case "nodeVersionChanged":
|
||||
goman.New(func() {
|
||||
sharedUpgradeManager.Start()
|
||||
})
|
||||
if !sharedUpgradeManager.IsInstalling() {
|
||||
goman.New(func() {
|
||||
sharedUpgradeManager.Start()
|
||||
})
|
||||
}
|
||||
case "scriptsChanged":
|
||||
err = this.reloadCommonScripts()
|
||||
if err != nil {
|
||||
@@ -345,7 +379,7 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -354,7 +388,7 @@ func (this *Node) loop() error {
|
||||
return err
|
||||
}
|
||||
case "nodeLevelChanged":
|
||||
levelInfoResp, err := rpcClient.NodeRPC().FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
|
||||
levelInfoResp, err := rpcClient.NodeRPC.FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -371,7 +405,7 @@ func (this *Node) loop() error {
|
||||
sharedNodeConfig.ParentNodes = parentNodes
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -380,7 +414,7 @@ func (this *Node) loop() error {
|
||||
return err
|
||||
}
|
||||
case "ddosProtectionChanged":
|
||||
resp, err := rpcClient.NodeRPC().FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
|
||||
resp, err := rpcClient.NodeRPC.FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -407,7 +441,7 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -452,7 +486,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
|
||||
nodeCtx := rpcClient.Context()
|
||||
|
||||
// TODO 这里考虑只同步版本号有变更的
|
||||
configResp, err := rpcClient.NodeRPC().FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
|
||||
configResp, err := rpcClient.NodeRPC.FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
|
||||
Version: -1, // 更新所有版本
|
||||
Compress: true,
|
||||
NodeTaskVersion: taskVersion,
|
||||
@@ -543,7 +577,7 @@ func (this *Node) syncServerConfig(serverId int64) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.ServerRPC().ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
|
||||
resp, err := rpcClient.ServerRPC.ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -605,7 +639,7 @@ func (this *Node) startSyncTimer() {
|
||||
// 检查集群设置
|
||||
func (this *Node) checkClusterConfig() error {
|
||||
configFile := Tea.ConfigFile("cluster.yaml")
|
||||
data, err := ioutil.ReadFile(configFile)
|
||||
data, err := os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -625,7 +659,7 @@ func (this *Node) checkClusterConfig() error {
|
||||
}
|
||||
|
||||
logs.Println("[NODE]registering node to cluster ...")
|
||||
resp, err := rpcClient.NodeRPC().RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
|
||||
resp, err := rpcClient.NodeRPC.RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -675,7 +709,7 @@ func (this *Node) listenSock() error {
|
||||
if this.sock.IsListening() {
|
||||
reply, err := this.sock.Send(&gosock.Command{Code: "pid"})
|
||||
if err == nil {
|
||||
return errors.New("error: the process is already running, pid: " + maps.NewMap(reply.Params).GetString("pid"))
|
||||
return errors.New("error: the process is already running, pid: " + types.String(maps.NewMap(reply.Params).GetInt("pid")))
|
||||
} else {
|
||||
return errors.New("error: the process is already running")
|
||||
}
|
||||
@@ -713,6 +747,7 @@ func (this *Node) listenSock() error {
|
||||
_ = this.sock.Close()
|
||||
|
||||
events.Notify(events.EventQuit)
|
||||
events.Notify(events.EventTerminated)
|
||||
|
||||
// 监控连接数,如果连接数为0,则退出进程
|
||||
goman.New(func() {
|
||||
@@ -763,20 +798,24 @@ func (this *Node) listenSock() error {
|
||||
},
|
||||
})
|
||||
case "conns":
|
||||
ipConns, serverConns := sharedClientConnLimiter.Conns()
|
||||
var addrs = []string{}
|
||||
var connMap = conns.SharedMap.AllConns()
|
||||
for _, conn := range connMap {
|
||||
addrs = append(addrs, conn.RemoteAddr().String())
|
||||
}
|
||||
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]interface{}{
|
||||
"ipConns": ipConns,
|
||||
"serverConns": serverConns,
|
||||
"total": sharedListenerManager.TotalActiveConnections(),
|
||||
"addrs": addrs,
|
||||
"total": len(addrs),
|
||||
},
|
||||
})
|
||||
case "dropIP":
|
||||
var m = maps.NewMap(cmd.Params)
|
||||
var ip = m.GetString("ip")
|
||||
var timeSeconds = m.GetInt("timeoutSeconds")
|
||||
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds)
|
||||
var async = m.GetBool("async")
|
||||
err := firewalls.Firewall().DropSourceIP(ip, timeSeconds, async)
|
||||
if err != nil {
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]interface{}{
|
||||
@@ -840,6 +879,11 @@ func (this *Node) listenSock() error {
|
||||
} else {
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
case "bandwidth":
|
||||
var m = stats.SharedBandwidthStatManager.Map()
|
||||
_ = cmd.Reply(&gosock.Command{Params: maps.Map{
|
||||
"stats": m,
|
||||
}})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -850,7 +894,7 @@ func (this *Node) listenSock() error {
|
||||
})
|
||||
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
logs.Println("NODE", "quit unix sock")
|
||||
remotelogs.Println("NODE", "quit unix sock")
|
||||
_ = this.sock.Close()
|
||||
})
|
||||
|
||||
@@ -994,19 +1038,22 @@ func (this *Node) reloadServer() {
|
||||
}
|
||||
|
||||
func (this *Node) checkDisk() {
|
||||
if runtime.GOOS == "linux" {
|
||||
if runtime.GOOS != "linux" {
|
||||
return
|
||||
}
|
||||
for n := 'a'; n <= 'z'; n++ {
|
||||
for _, path := range []string{
|
||||
"/sys/block/vda/queue/rotational",
|
||||
"/sys/block/sda/queue/rotational",
|
||||
"/sys/block/vd" + string(n) + "/queue/rotational",
|
||||
"/sys/block/sd" + string(n) + "/queue/rotational",
|
||||
} {
|
||||
data, err := ioutil.ReadFile(path)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if string(data) == "0" {
|
||||
teaconst.DiskIsFast = true
|
||||
}
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
@@ -25,7 +24,7 @@ func (this *Node) handlePanic() {
|
||||
var panicFile = Tea.Root + "/logs/panic.log"
|
||||
|
||||
// 分析panic
|
||||
data, err := ioutil.ReadFile(panicFile)
|
||||
data, err := os.ReadFile(panicFile)
|
||||
if err == nil {
|
||||
var index = bytes.Index(data, []byte("panic:"))
|
||||
if index >= 0 {
|
||||
|
||||
@@ -30,10 +30,14 @@ type NodeStatusExecutor struct {
|
||||
cpuUpdatedTime time.Time
|
||||
cpuLogicalCount int
|
||||
cpuPhysicalCount int
|
||||
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
func NewNodeStatusExecutor() *NodeStatusExecutor {
|
||||
return &NodeStatusExecutor{}
|
||||
return &NodeStatusExecutor{
|
||||
ticker: time.NewTicker(30 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *NodeStatusExecutor) Listen() {
|
||||
@@ -41,15 +45,12 @@ func (this *NodeStatusExecutor) Listen() {
|
||||
this.cpuUpdatedTime = time.Now()
|
||||
this.update()
|
||||
|
||||
// TODO 这个时间间隔可以配置
|
||||
var ticker = time.NewTicker(30 * time.Second)
|
||||
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
remotelogs.Println("NODE_STATUS", "quit executor")
|
||||
ticker.Stop()
|
||||
this.ticker.Stop()
|
||||
})
|
||||
|
||||
for range ticker.C {
|
||||
for range this.ticker.C {
|
||||
this.isFirstTime = false
|
||||
this.update()
|
||||
}
|
||||
@@ -124,7 +125,7 @@ func (this *NodeStatusExecutor) update() {
|
||||
remotelogs.Error("NODE_STATUS", "failed to open rpc: "+err.Error())
|
||||
return
|
||||
}
|
||||
_, err = rpcClient.NodeRPC().UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
|
||||
_, err = rpcClient.NodeRPC.UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
|
||||
StatusJSON: jsonData,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
@@ -104,7 +103,7 @@ func (this *SystemServiceManager) setupSystemd(params maps.Map) error {
|
||||
|
||||
if output == "enabled" {
|
||||
// 检查文件路径是否变化
|
||||
data, err := ioutil.ReadFile("/etc/systemd/system/" + teaconst.SystemdServiceName + ".service")
|
||||
data, err := os.ReadFile("/etc/systemd/system/" + teaconst.SystemdServiceName + ".service")
|
||||
if err == nil && bytes.Index(data, []byte(exe)) > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ func (this *OCSPUpdateTask) Loop() error {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := rpcClient.SSLCertRPC().ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
|
||||
resp, err := rpcClient.SSLCertRPC.ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
|
||||
Version: this.version,
|
||||
Size: 100,
|
||||
})
|
||||
|
||||
@@ -81,7 +81,7 @@ func (this *SyncAPINodesTask) Loop() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.APINodeRPC().FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
|
||||
resp, err := rpcClient.APINodeRPC.FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -14,8 +14,10 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"github.com/iwind/gosock/pkg/gosock"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
@@ -46,15 +48,12 @@ func (this *UpgradeManager) Start() {
|
||||
}
|
||||
this.isInstalling = true
|
||||
|
||||
// 还原安装状态
|
||||
defer func() {
|
||||
this.isInstalling = false
|
||||
}()
|
||||
|
||||
remotelogs.Println("UPGRADE_MANAGER", "upgrading node ...")
|
||||
err := this.install()
|
||||
if err != nil {
|
||||
remotelogs.Error("UPGRADE_MANAGER", "download failed: "+err.Error())
|
||||
|
||||
this.isInstalling = false
|
||||
return
|
||||
}
|
||||
|
||||
@@ -65,9 +64,16 @@ func (this *UpgradeManager) Start() {
|
||||
if err != nil {
|
||||
remotelogs.Error("UPGRADE_MANAGER", err.Error())
|
||||
}
|
||||
|
||||
this.isInstalling = false
|
||||
})
|
||||
}
|
||||
|
||||
// IsInstalling 检查是否正在安装
|
||||
func (this *UpgradeManager) IsInstalling() bool {
|
||||
return this.isInstalling
|
||||
}
|
||||
|
||||
func (this *UpgradeManager) install() error {
|
||||
// 检查是否有已下载但未安装成功的
|
||||
if len(this.lastFile) > 0 {
|
||||
@@ -83,7 +89,7 @@ func (this *UpgradeManager) install() error {
|
||||
}
|
||||
|
||||
// 创建临时文件
|
||||
dir := Tea.Root + "/tmp"
|
||||
var dir = Tea.Root + "/tmp"
|
||||
_, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -98,7 +104,7 @@ func (this *UpgradeManager) install() error {
|
||||
|
||||
remotelogs.Println("UPGRADE_MANAGER", "downloading new node ...")
|
||||
|
||||
path := dir + "/edge-node" + ".tmp"
|
||||
var path = dir + "/edge-node" + ".tmp"
|
||||
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -120,7 +126,7 @@ func (this *UpgradeManager) install() error {
|
||||
var sum = ""
|
||||
var filename = ""
|
||||
for {
|
||||
resp, err := client.NodeRPC().DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
|
||||
resp, err := client.NodeRPC.DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
|
||||
Os: runtime.GOOS,
|
||||
Arch: runtime.GOARCH,
|
||||
ChunkOffset: offset,
|
||||
@@ -203,16 +209,16 @@ func (this *UpgradeManager) unzip(zipPath string) error {
|
||||
}
|
||||
|
||||
// 先改先前的可执行文件
|
||||
err := os.Rename(target+"/bin/edge-node", target+"/bin/.edge-node.dist")
|
||||
hasBackup := err == nil
|
||||
err := os.Rename(target+"/bin/"+teaconst.ProcessName, target+"/bin/."+teaconst.ProcessName+".dist")
|
||||
var hasBackup = err == nil
|
||||
defer func() {
|
||||
if !isOk && hasBackup {
|
||||
// 失败时还原
|
||||
_ = os.Rename(target+"/bin/.edge-node.dist", target+"/bin/edge-node")
|
||||
_ = os.Rename(target+"/bin/."+teaconst.ProcessName+".dist", target+"/bin/"+teaconst.ProcessName)
|
||||
}
|
||||
}()
|
||||
|
||||
unzip := utils.NewUnzip(zipPath, target, "edge-node/")
|
||||
var unzip = utils.NewUnzip(zipPath, target, "edge-node/")
|
||||
err = unzip.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -225,6 +231,9 @@ func (this *UpgradeManager) unzip(zipPath string) error {
|
||||
|
||||
// 重启
|
||||
func (this *UpgradeManager) restart() error {
|
||||
// 关闭当前sock,防止无法重启
|
||||
_ = gosock.NewTmpSock(teaconst.ProcessName).Close()
|
||||
|
||||
// 重新启动
|
||||
if DaemonIsOn && DaemonPid == os.Getppid() {
|
||||
utils.Exit() // TODO 试着更优雅重启
|
||||
@@ -241,7 +250,9 @@ func (this *UpgradeManager) restart() error {
|
||||
events.Notify(events.EventTerminated)
|
||||
|
||||
// 启动
|
||||
cmd := exec.Command(exe, "start")
|
||||
exe = filepath.Dir(exe) + "/" + teaconst.ProcessName
|
||||
|
||||
var cmd = exec.Command(exe, "start")
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -109,6 +109,8 @@ func TestRegexp_ParseKeywords3(t *testing.T) {
|
||||
|
||||
func BenchmarkRegexp_MatchString(b *testing.B) {
|
||||
var r = re.MustCompile("(?i)(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)(\\s|%09|%0A|(\\+|%20))*(=|%3D)")
|
||||
b.ResetTimer()
|
||||
|
||||
//b.Log("keywords:", r.Keywords())
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
@@ -117,6 +119,8 @@ func BenchmarkRegexp_MatchString(b *testing.B) {
|
||||
|
||||
func BenchmarkRegexp_MatchString2(b *testing.B) {
|
||||
var r = regexp.MustCompile("(?i)(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)(\\s|%09|%0A|(\\+|%20))*(=|%3D)")
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
}
|
||||
|
||||
@@ -173,7 +173,6 @@ func ServerSuccess(serverId int64, tag string, description string, logType nodec
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ServerLog 打印服务相关日志信息
|
||||
func ServerLog(serverId int64, tag string, description string, logType nodeconfigs.NodeLogType, params maps.Map) {
|
||||
logs.Println("[" + tag + "]" + description)
|
||||
@@ -253,6 +252,6 @@ Loop:
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = rpcClient.NodeLogRPC().CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
|
||||
_, err = rpcClient.NodeLogRPC.CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -28,6 +28,27 @@ type RPCClient struct {
|
||||
conns []*grpc.ClientConn
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
NodeRPC pb.NodeServiceClient
|
||||
NodeLogRPC pb.NodeLogServiceClient
|
||||
NodeTaskRPC pb.NodeTaskServiceClient
|
||||
NodeValueRPC pb.NodeValueServiceClient
|
||||
HTTPAccessLogRPC pb.HTTPAccessLogServiceClient
|
||||
HTTPCacheTaskKeyRPC pb.HTTPCacheTaskKeyServiceClient
|
||||
APINodeRPC pb.APINodeServiceClient
|
||||
IPLibraryArtifactRPC pb.IPLibraryArtifactServiceClient
|
||||
IPListRPC pb.IPListServiceClient
|
||||
IPItemRPC pb.IPItemServiceClient
|
||||
FileRPC pb.FileServiceClient
|
||||
FileChunkRPC pb.FileChunkServiceClient
|
||||
ACMEAuthenticationRPC pb.ACMEAuthenticationServiceClient
|
||||
ServerRPC pb.ServerServiceClient
|
||||
ServerDailyStatRPC pb.ServerDailyStatServiceClient
|
||||
ServerBandwidthStatRPC pb.ServerBandwidthStatServiceClient
|
||||
MetricStatRPC pb.MetricStatServiceClient
|
||||
FirewallRPC pb.FirewallServiceClient
|
||||
SSLCertRPC pb.SSLCertServiceClient
|
||||
ScriptRPC pb.ScriptServiceClient
|
||||
}
|
||||
|
||||
func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
|
||||
@@ -35,10 +56,32 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
|
||||
return nil, errors.New("api config should not be nil")
|
||||
}
|
||||
|
||||
client := &RPCClient{
|
||||
var client = &RPCClient{
|
||||
apiConfig: apiConfig,
|
||||
}
|
||||
|
||||
// 初始化RPC实例
|
||||
client.NodeRPC = pb.NewNodeServiceClient(client)
|
||||
client.NodeLogRPC = pb.NewNodeLogServiceClient(client)
|
||||
client.NodeTaskRPC = pb.NewNodeTaskServiceClient(client)
|
||||
client.NodeValueRPC = pb.NewNodeValueServiceClient(client)
|
||||
client.HTTPAccessLogRPC = pb.NewHTTPAccessLogServiceClient(client)
|
||||
client.HTTPCacheTaskKeyRPC = pb.NewHTTPCacheTaskKeyServiceClient(client)
|
||||
client.APINodeRPC = pb.NewAPINodeServiceClient(client)
|
||||
client.IPLibraryArtifactRPC = pb.NewIPLibraryArtifactServiceClient(client)
|
||||
client.IPListRPC = pb.NewIPListServiceClient(client)
|
||||
client.IPItemRPC = pb.NewIPItemServiceClient(client)
|
||||
client.FileRPC = pb.NewFileServiceClient(client)
|
||||
client.FileChunkRPC = pb.NewFileChunkServiceClient(client)
|
||||
client.ACMEAuthenticationRPC = pb.NewACMEAuthenticationServiceClient(client)
|
||||
client.ServerRPC = pb.NewServerServiceClient(client)
|
||||
client.ServerDailyStatRPC = pb.NewServerDailyStatServiceClient(client)
|
||||
client.ServerBandwidthStatRPC = pb.NewServerBandwidthStatServiceClient(client)
|
||||
client.MetricStatRPC = pb.NewMetricStatServiceClient(client)
|
||||
client.FirewallRPC = pb.NewFirewallServiceClient(client)
|
||||
client.SSLCertRPC = pb.NewSSLCertServiceClient(client)
|
||||
client.ScriptRPC = pb.NewScriptServiceClient(client)
|
||||
|
||||
err := client.init()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -47,102 +90,10 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeRPC() pb.NodeServiceClient {
|
||||
return pb.NewNodeServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeLogRPC() pb.NodeLogServiceClient {
|
||||
return pb.NewNodeLogServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeTaskRPC() pb.NodeTaskServiceClient {
|
||||
return pb.NewNodeTaskServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeValueRPC() pb.NodeValueServiceClient {
|
||||
return pb.NewNodeValueServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) HTTPAccessLogRPC() pb.HTTPAccessLogServiceClient {
|
||||
return pb.NewHTTPAccessLogServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) HTTPCacheTaskKeyRPC() pb.HTTPCacheTaskKeyServiceClient {
|
||||
return pb.NewHTTPCacheTaskKeyServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) APINodeRPC() pb.APINodeServiceClient {
|
||||
return pb.NewAPINodeServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPLibraryRPC() pb.IPLibraryServiceClient {
|
||||
return pb.NewIPLibraryServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionCountryRPC() pb.RegionCountryServiceClient {
|
||||
return pb.NewRegionCountryServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionProvinceRPC() pb.RegionProvinceServiceClient {
|
||||
return pb.NewRegionProvinceServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionCityRPC() pb.RegionCityServiceClient {
|
||||
return pb.NewRegionCityServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionProviderRPC() pb.RegionProviderServiceClient {
|
||||
return pb.NewRegionProviderServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPListRPC() pb.IPListServiceClient {
|
||||
return pb.NewIPListServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPItemRPC() pb.IPItemServiceClient {
|
||||
return pb.NewIPItemServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) FileRPC() pb.FileServiceClient {
|
||||
return pb.NewFileServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) FileChunkRPC() pb.FileChunkServiceClient {
|
||||
return pb.NewFileChunkServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ACMEAuthenticationRPC() pb.ACMEAuthenticationServiceClient {
|
||||
return pb.NewACMEAuthenticationServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ServerRPC() pb.ServerServiceClient {
|
||||
return pb.NewServerServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient {
|
||||
return pb.NewServerDailyStatServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ServerBandwidthStatRPC() pb.ServerBandwidthStatServiceClient {
|
||||
return pb.NewServerBandwidthStatServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) MetricStatRPC() pb.MetricStatServiceClient {
|
||||
return pb.NewMetricStatServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) FirewallRPC() pb.FirewallServiceClient {
|
||||
return pb.NewFirewallServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) SSLCertRPC() pb.SSLCertServiceClient {
|
||||
return pb.NewSSLCertServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
// Context 节点上下文信息
|
||||
func (this *RPCClient) Context() context.Context {
|
||||
ctx := context.Background()
|
||||
m := maps.Map{
|
||||
var ctx = context.Background()
|
||||
var m = maps.Map{
|
||||
"timestamp": time.Now().Unix(),
|
||||
"type": "node",
|
||||
"userId": 0,
|
||||
@@ -157,7 +108,7 @@ func (this *RPCClient) Context() context.Context {
|
||||
utils.PrintError(err)
|
||||
return context.Background()
|
||||
}
|
||||
token := base64.StdEncoding.EncodeToString(data)
|
||||
var token = base64.StdEncoding.EncodeToString(data)
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, "nodeId", this.apiConfig.NodeId, "token", token)
|
||||
return ctx
|
||||
}
|
||||
@@ -209,7 +160,7 @@ func (this *RPCClient) UpdateConfig(config *configs.APIConfig) error {
|
||||
// 初始化
|
||||
func (this *RPCClient) init() error {
|
||||
// 重新连接
|
||||
conns := []*grpc.ClientConn{}
|
||||
var conns = []*grpc.ClientConn{}
|
||||
for _, endpoint := range this.apiConfig.RPC.Endpoints {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
@@ -248,20 +199,24 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
|
||||
|
||||
// 检查连接状态
|
||||
if len(this.conns) > 0 {
|
||||
availableConns := []*grpc.ClientConn{}
|
||||
for _, state := range []connectivity.State{connectivity.Ready, connectivity.Idle, connectivity.Connecting} {
|
||||
var availableConns = []*grpc.ClientConn{}
|
||||
for _, stateArray := range [][2]connectivity.State{
|
||||
{connectivity.Ready, connectivity.Idle}, // 优先Ready和Idle
|
||||
{connectivity.Connecting, connectivity.Connecting},
|
||||
} {
|
||||
for _, conn := range this.conns {
|
||||
if conn.GetState() == state {
|
||||
var state = conn.GetState()
|
||||
if state == stateArray[0] || state == stateArray[1] {
|
||||
availableConns = append(availableConns, conn)
|
||||
}
|
||||
}
|
||||
if len(availableConns) > 0 {
|
||||
break
|
||||
return this.randConn(availableConns)
|
||||
}
|
||||
}
|
||||
|
||||
if len(availableConns) > 0 {
|
||||
return availableConns[rands.Int(0, len(availableConns)-1)]
|
||||
return this.randConn(availableConns)
|
||||
}
|
||||
|
||||
// 关闭
|
||||
@@ -281,5 +236,31 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
|
||||
return nil
|
||||
}
|
||||
|
||||
return this.conns[rands.Int(0, len(this.conns)-1)]
|
||||
return this.randConn(this.conns)
|
||||
}
|
||||
|
||||
func (this *RPCClient) Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...grpc.CallOption) error {
|
||||
var conn = this.pickConn()
|
||||
if conn == nil {
|
||||
return errors.New("can not get available grpc connection")
|
||||
}
|
||||
return conn.Invoke(ctx, method, args, reply, opts...)
|
||||
}
|
||||
func (this *RPCClient) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
|
||||
var conn = this.pickConn()
|
||||
if conn == nil {
|
||||
return nil, errors.New("can not get available grpc connection")
|
||||
}
|
||||
return conn.NewStream(ctx, desc, method, opts...)
|
||||
}
|
||||
|
||||
func (this *RPCClient) randConn(conns []*grpc.ClientConn) *grpc.ClientConn {
|
||||
var l = len(conns)
|
||||
if l == 0 {
|
||||
return nil
|
||||
}
|
||||
if l == 1 {
|
||||
return conns[0]
|
||||
}
|
||||
return conns[rands.Int(0, l-1)]
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user