Compare commits
42 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57cb00edf0 | ||
|
|
3fb39b479a | ||
|
|
4a1daff143 | ||
|
|
dd1dbd424e | ||
|
|
305cb4b46e | ||
|
|
ef90dce29b | ||
|
|
3cb69f4c71 | ||
|
|
af4cd05df2 | ||
|
|
64e0ae80b7 | ||
|
|
8bba228745 | ||
|
|
8cc06e6707 | ||
|
|
52fdee2eeb | ||
|
|
b5f52dd136 | ||
|
|
abda886de5 | ||
|
|
18f08525b9 | ||
|
|
b2a9a31fe5 | ||
|
|
f578114aeb | ||
|
|
bf4f47fc35 | ||
|
|
0be951742a | ||
|
|
59d3d6ae4b | ||
|
|
d061876f7e | ||
|
|
ddaec82415 | ||
|
|
8afd00f00d | ||
|
|
0b306f0a22 | ||
|
|
b36a36172b | ||
|
|
770278bbbc | ||
|
|
ca24818571 | ||
|
|
cd0af22655 | ||
|
|
96cb8d8af7 | ||
|
|
91face15bf | ||
|
|
6f52df63a5 | ||
|
|
509d81dc66 | ||
|
|
817f2a6f91 | ||
|
|
d74e10c7a8 | ||
|
|
1a1280b76e | ||
|
|
85d3b1169f | ||
|
|
1b32292e0c | ||
|
|
e6ba44fb2f | ||
|
|
9b6e022f46 | ||
|
|
4e9ab19fc0 | ||
|
|
4d634d8fa5 | ||
|
|
a538282e4f |
@@ -52,7 +52,6 @@ function build() {
|
||||
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
|
||||
cp -R "$ROOT"/www "$DIST"/
|
||||
cp -R "$ROOT"/pages "$DIST"/
|
||||
cp -R "$ROOT"/resources "$DIST"/
|
||||
|
||||
# we support TOA on linux/amd64 only
|
||||
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]
|
||||
|
||||
Binary file not shown.
@@ -318,6 +318,21 @@ func main() {
|
||||
}
|
||||
}
|
||||
})
|
||||
app.On("bandwidth", func() {
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
return
|
||||
}
|
||||
var statsMap = maps.NewMap(reply.Params).Get("stats")
|
||||
statsJSON, err := json.MarshalIndent(statsMap, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]" + err.Error())
|
||||
return
|
||||
}
|
||||
fmt.Println(string(statsJSON))
|
||||
})
|
||||
app.Run(func() {
|
||||
var node = nodes.NewNode()
|
||||
node.Start()
|
||||
|
||||
11
go.mod
11
go.mod
@@ -4,8 +4,7 @@ go 1.18
|
||||
|
||||
replace (
|
||||
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
|
||||
github.com/fsnotify/fsnotify => /Users/WorkSpace/Projects/fsnotify
|
||||
rogchap.com/v8go => /Users/Workspace/Projects/v8go
|
||||
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -21,7 +20,7 @@ require (
|
||||
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
|
||||
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5
|
||||
github.com/klauspost/compress v1.15.8
|
||||
github.com/mattn/go-sqlite3 v1.14.9
|
||||
github.com/mdlayher/netlink v1.4.2
|
||||
@@ -29,9 +28,9 @@ require (
|
||||
github.com/mssola/user_agent v0.5.3
|
||||
github.com/pires/go-proxyproto v0.6.1
|
||||
github.com/shirou/gopsutil/v3 v3.22.2
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
|
||||
golang.org/x/text v0.3.7
|
||||
google.golang.org/grpc v1.45.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -41,7 +40,7 @@ require (
|
||||
require (
|
||||
github.com/BurntSushi/toml v0.4.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/chai2010/webp v1.1.0 // indirect
|
||||
github.com/chai2010/webp v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-sql-driver/mysql v1.5.0 // indirect
|
||||
|
||||
77
go.sum
77
go.sum
@@ -1,17 +1,13 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
|
||||
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
|
||||
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
@@ -20,8 +16,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
|
||||
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
|
||||
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
|
||||
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
|
||||
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
|
||||
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
@@ -37,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
|
||||
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -47,15 +42,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
|
||||
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
|
||||
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
@@ -87,22 +76,20 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
|
||||
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/iwind/TeaGo v0.0.0-20220807023459-448081424640 h1:nBVQzDI4mrQS+Egg+Li6BGiTToBsv+XTck+BItgI52k=
|
||||
github.com/iwind/TeaGo v0.0.0-20220807023459-448081424640/go.mod h1:+K2l6Num4Evl0jH7TYlZJ1oFJX8sA8YUC31Pb+I1mJk=
|
||||
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55 h1:shQNx0flJFBwKsGE7Hs3bI2bDz+YF0zl/4qE8B2KRiY=
|
||||
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
|
||||
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
|
||||
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
|
||||
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
|
||||
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
|
||||
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
|
||||
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5 h1:SgKhrqRhWVpu0ZKxQM8MGjdhy7hlH3Xax8snwsZWSic=
|
||||
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
|
||||
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
|
||||
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
|
||||
@@ -119,7 +106,6 @@ github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed
|
||||
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
|
||||
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
|
||||
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -155,19 +141,9 @@ github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
|
||||
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
|
||||
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
|
||||
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
|
||||
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -182,7 +158,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
|
||||
@@ -196,41 +171,30 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
|
||||
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
|
||||
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
@@ -259,22 +223,15 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -292,7 +249,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -302,11 +258,10 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
@@ -318,7 +273,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
|
||||
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
|
||||
@@ -332,7 +286,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
|
||||
@@ -341,7 +294,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
|
||||
@@ -360,22 +312,13 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@@ -96,13 +96,13 @@ func (this *FileList) Reset() error {
|
||||
}
|
||||
|
||||
func (this *FileList) Add(hash string, item *Item) error {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := db.Add(hash, item)
|
||||
err := db.AddAsync(hash, item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
|
||||
}
|
||||
|
||||
func (this *FileList) Exist(hash string) (bool, error) {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// 如果Hash列表里不存在,那么必然不存在
|
||||
if !db.hashMap.Exist(hash) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var item = this.memoryCache.Read(hash)
|
||||
if item != nil {
|
||||
return true, nil
|
||||
@@ -225,7 +230,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
|
||||
return err
|
||||
}
|
||||
if notFound {
|
||||
_, err = db.deleteHitByHashStmt.Exec(hash)
|
||||
err = db.DeleteHitAsync(hash)
|
||||
if err != nil {
|
||||
return db.WrapError(err)
|
||||
}
|
||||
@@ -266,7 +271,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
|
||||
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
|
||||
_ = check
|
||||
|
||||
row := db.statStmt.QueryRow()
|
||||
var row = db.statStmt.QueryRow()
|
||||
if row.Err() != nil {
|
||||
return nil, row.Err()
|
||||
}
|
||||
@@ -291,13 +296,13 @@ func (this *FileList) Count() (int64, error) {
|
||||
|
||||
// IncreaseHit 增加点击量
|
||||
func (this *FileList) IncreaseHit(hash string) error {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return db.IncreaseHit(hash)
|
||||
return db.IncreaseHitAsync(hash)
|
||||
}
|
||||
|
||||
// OnAdd 添加事件
|
||||
@@ -326,17 +331,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
|
||||
return fnv.HashString(hash) % CountFileDB
|
||||
}
|
||||
|
||||
func (this *FileList) getDB(hash string) *FileListDB {
|
||||
func (this *FileList) GetDB(hash string) *FileListDB {
|
||||
return this.dbList[fnv.HashString(hash)%CountFileDB]
|
||||
}
|
||||
|
||||
func (this *FileList) remove(hash string) (notFound bool, err error) {
|
||||
var db = this.getDB(hash)
|
||||
var db = this.GetDB(hash)
|
||||
|
||||
if !db.IsReady() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// HashMap中不存在,则确定不存在
|
||||
if !db.hashMap.Exist(hash) {
|
||||
return true, nil
|
||||
}
|
||||
defer db.hashMap.Delete(hash)
|
||||
|
||||
// 从缓存中删除
|
||||
this.memoryCache.Delete(hash)
|
||||
|
||||
@@ -357,14 +368,14 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = db.deleteByHashStmt.Exec(hash)
|
||||
err = db.DeleteAsync(hash)
|
||||
if err != nil {
|
||||
return false, db.WrapError(err)
|
||||
}
|
||||
|
||||
atomic.AddInt64(&this.total, -1)
|
||||
|
||||
_, err = db.deleteHitByHashStmt.Exec(hash)
|
||||
err = db.DeleteHitAsync(hash)
|
||||
if err != nil {
|
||||
return false, db.WrapError(err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
@@ -21,6 +23,10 @@ type FileListDB struct {
|
||||
readDB *dbs.DB
|
||||
writeDB *dbs.DB
|
||||
|
||||
writeBatch *dbs.Batch
|
||||
|
||||
hashMap *FileListHashMap
|
||||
|
||||
itemsTableName string
|
||||
hitsTableName string
|
||||
|
||||
@@ -30,31 +36,48 @@ type FileListDB struct {
|
||||
isReady bool
|
||||
|
||||
// cacheItems
|
||||
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
|
||||
insertStmt *dbs.Stmt // 写入数据
|
||||
selectByHashStmt *dbs.Stmt // 使用hash查询数据
|
||||
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
|
||||
|
||||
insertStmt *dbs.Stmt // 写入数据
|
||||
insertSQL string
|
||||
|
||||
selectByHashStmt *dbs.Stmt // 使用hash查询数据
|
||||
|
||||
selectHashListStmt *dbs.Stmt
|
||||
|
||||
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
deleteByHashSQL string
|
||||
|
||||
statStmt *dbs.Stmt // 统计
|
||||
purgeStmt *dbs.Stmt // 清理
|
||||
deleteAllStmt *dbs.Stmt // 删除所有数据
|
||||
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
|
||||
|
||||
// hits
|
||||
insertHitStmt *dbs.Stmt // 写入数据
|
||||
increaseHitStmt *dbs.Stmt // 增加点击量
|
||||
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
|
||||
lfuHitsStmt *dbs.Stmt // 读取老的数据
|
||||
insertHitSQL string // 写入数据
|
||||
increaseHitSQL string // 增加点击量
|
||||
deleteHitByHashSQL string // 根据hash删除数据
|
||||
lfuHitsStmt *dbs.Stmt // 读取老的数据
|
||||
}
|
||||
|
||||
func NewFileListDB() *FileListDB {
|
||||
return &FileListDB{}
|
||||
return &FileListDB{
|
||||
hashMap: NewFileListHashMap(),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListDB) Open(dbPath string) error {
|
||||
this.dbPath = dbPath
|
||||
|
||||
// 动态调整Cache值
|
||||
var cacheSize = 32000
|
||||
var memoryGB = utils.SystemMemoryGB()
|
||||
if memoryGB >= 8 {
|
||||
cacheSize += 32000 * memoryGB / 8
|
||||
}
|
||||
|
||||
// write db
|
||||
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
|
||||
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize)+"&_secure_delete=FAST")
|
||||
if err != nil {
|
||||
return errors.New("open write database failed: " + err.Error())
|
||||
}
|
||||
@@ -69,13 +92,22 @@ func (this *FileListDB) Open(dbPath string) error {
|
||||
}**/
|
||||
|
||||
this.writeDB = dbs.NewDB(writeDB)
|
||||
this.writeBatch = dbs.NewBatch(writeDB, 4)
|
||||
this.writeBatch.OnFail(func(err error) {
|
||||
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error())
|
||||
})
|
||||
|
||||
goman.New(func() {
|
||||
this.writeBatch.Exec()
|
||||
})
|
||||
|
||||
if teaconst.EnableDBStat {
|
||||
this.writeBatch.EnableStat(true)
|
||||
this.writeDB.EnableStat(true)
|
||||
}
|
||||
|
||||
// read db
|
||||
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
|
||||
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize))
|
||||
if err != nil {
|
||||
return errors.New("open read database failed: " + err.Error())
|
||||
}
|
||||
@@ -119,7 +151,8 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -129,7 +162,10 @@ func (this *FileListDB) Init() error {
|
||||
return err
|
||||
}
|
||||
|
||||
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
|
||||
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
|
||||
|
||||
this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
|
||||
this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -151,17 +187,11 @@ func (this *FileListDB) Init() error {
|
||||
|
||||
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
|
||||
|
||||
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
|
||||
this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
|
||||
|
||||
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
|
||||
|
||||
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
|
||||
|
||||
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
|
||||
if err != nil {
|
||||
@@ -170,6 +200,14 @@ func (this *FileListDB) Init() error {
|
||||
|
||||
this.isReady = true
|
||||
|
||||
// 加载HashMap
|
||||
go func() {
|
||||
err := this.hashMap.Load(this)
|
||||
if err != nil {
|
||||
remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -181,12 +219,25 @@ func (this *FileListDB) Total() int64 {
|
||||
return this.total
|
||||
}
|
||||
|
||||
func (this *FileListDB) Add(hash string, item *Item) error {
|
||||
func (this *FileListDB) AddAsync(hash string, item *Item) error {
|
||||
this.hashMap.Add(hash)
|
||||
|
||||
if item.StaleAt == 0 {
|
||||
item.StaleAt = item.ExpiredAt
|
||||
}
|
||||
|
||||
this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (this *FileListDB) AddSync(hash string, item *Item) error {
|
||||
this.hashMap.Add(hash)
|
||||
|
||||
if item.StaleAt == 0 {
|
||||
item.StaleAt = item.ExpiredAt
|
||||
}
|
||||
|
||||
// 放入队列
|
||||
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
|
||||
if err != nil {
|
||||
return this.WrapError(err)
|
||||
@@ -195,6 +246,23 @@ func (this *FileListDB) Add(hash string, item *Item) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteAsync(hash string) error {
|
||||
this.hashMap.Delete(hash)
|
||||
|
||||
this.writeBatch.Add(this.deleteByHashSQL, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteSync(hash string) error {
|
||||
this.hashMap.Delete(hash)
|
||||
|
||||
_, err := this.deleteByHashStmt.Exec(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) {
|
||||
if !this.isReady {
|
||||
return nil, nil
|
||||
@@ -250,10 +318,36 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHit(hash string) error {
|
||||
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
|
||||
rows, err := this.selectHashListStmt.Query(lastId)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var id int64
|
||||
var hash string
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&id, &hash)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return
|
||||
}
|
||||
maxId = id
|
||||
hashList = append(hashList, hash)
|
||||
}
|
||||
|
||||
_ = rows.Close()
|
||||
return
|
||||
}
|
||||
|
||||
func (this *FileListDB) IncreaseHitAsync(hash string) error {
|
||||
var week = timeutil.Format("YW")
|
||||
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
|
||||
return this.WrapError(err)
|
||||
this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) DeleteHitAsync(hash string) error {
|
||||
this.writeBatch.Add(this.deleteHitByHashSQL, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListDB) CleanPrefix(prefix string) error {
|
||||
@@ -288,6 +382,8 @@ func (this *FileListDB) CleanAll() error {
|
||||
return this.WrapError(err)
|
||||
}
|
||||
|
||||
this.hashMap.Clean()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -304,6 +400,9 @@ func (this *FileListDB) Close() error {
|
||||
if this.selectByHashStmt != nil {
|
||||
_ = this.selectByHashStmt.Close()
|
||||
}
|
||||
if this.selectHashListStmt != nil {
|
||||
_ = this.selectHashListStmt.Close()
|
||||
}
|
||||
if this.deleteByHashStmt != nil {
|
||||
_ = this.deleteByHashStmt.Close()
|
||||
}
|
||||
@@ -319,16 +418,6 @@ func (this *FileListDB) Close() error {
|
||||
if this.listOlderItemsStmt != nil {
|
||||
_ = this.listOlderItemsStmt.Close()
|
||||
}
|
||||
|
||||
if this.insertHitStmt != nil {
|
||||
_ = this.insertHitStmt.Close()
|
||||
}
|
||||
if this.increaseHitStmt != nil {
|
||||
_ = this.increaseHitStmt.Close()
|
||||
}
|
||||
if this.deleteHitByHashStmt != nil {
|
||||
_ = this.deleteHitByHashStmt.Close()
|
||||
}
|
||||
if this.lfuHitsStmt != nil {
|
||||
_ = this.lfuHitsStmt.Close()
|
||||
}
|
||||
@@ -349,6 +438,10 @@ func (this *FileListDB) Close() error {
|
||||
}
|
||||
}
|
||||
|
||||
if this.writeBatch != nil {
|
||||
this.writeBatch.Close()
|
||||
}
|
||||
|
||||
if len(errStrings) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
114
internal/caches/list_file_hash_map.go
Normal file
114
internal/caches/list_file_hash_map.go
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"math/big"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// FileListHashMap 文件Hash列表
|
||||
type FileListHashMap struct {
|
||||
m map[uint64]zero.Zero
|
||||
|
||||
locker sync.RWMutex
|
||||
isAvailable bool
|
||||
isReady bool
|
||||
}
|
||||
|
||||
func NewFileListHashMap() *FileListHashMap {
|
||||
return &FileListHashMap{
|
||||
m: map[uint64]zero.Zero{},
|
||||
isAvailable: false,
|
||||
isReady: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Load(db *FileListDB) error {
|
||||
// 如果系统内存过小,我们不缓存
|
||||
if utils.SystemMemoryGB() < 3 {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.isAvailable = true
|
||||
|
||||
var lastId int64
|
||||
for {
|
||||
hashList, maxId, err := db.ListHashes(lastId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(hashList) == 0 {
|
||||
break
|
||||
}
|
||||
this.AddHashes(hashList)
|
||||
lastId = maxId
|
||||
}
|
||||
|
||||
this.isReady = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Add(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
this.m[this.bigInt(hash)] = zero.New()
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) AddHashes(hashes []string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
for _, hash := range hashes {
|
||||
this.m[this.bigInt(hash)] = zero.New()
|
||||
}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Delete(hash string) {
|
||||
if !this.isAvailable {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
delete(this.m, this.bigInt(hash))
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Exist(hash string) bool {
|
||||
if !this.isAvailable {
|
||||
return true
|
||||
}
|
||||
if !this.isReady {
|
||||
// 只有完全Ready时才能判断是否为false
|
||||
return true
|
||||
}
|
||||
this.locker.RLock()
|
||||
_, ok := this.m[this.bigInt(hash)]
|
||||
this.locker.RUnlock()
|
||||
return ok
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) Clean() {
|
||||
this.locker.Lock()
|
||||
this.m = map[uint64]zero.Zero{}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) IsReady() bool {
|
||||
return this.isReady
|
||||
}
|
||||
|
||||
func (this *FileListHashMap) bigInt(hash string) uint64 {
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
return bigInt.Uint64()
|
||||
}
|
||||
91
internal/caches/list_file_hash_map_test.go
Normal file
91
internal/caches/list_file_hash_map_test.go
Normal file
@@ -0,0 +1,91 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
"math/big"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFileListHashMap_Memory(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m.Add(stringutil.Md5(types.String(i)))
|
||||
}
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Memory2(t *testing.T) {
|
||||
var stat1 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat1)
|
||||
|
||||
var m = map[uint64]zero.Zero{}
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
m[uint64(i)] = zero.New()
|
||||
}
|
||||
|
||||
var stat2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(stat2)
|
||||
|
||||
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
|
||||
}
|
||||
|
||||
func TestFileListHashMap_BigInt(t *testing.T) {
|
||||
for _, s := range []string{"1", "2", "3", "123", "123456"} {
|
||||
var hash = stringutil.Md5(s)
|
||||
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileListHashMap_Load(t *testing.T) {
|
||||
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = list.Close()
|
||||
}()
|
||||
|
||||
var m = caches.NewFileListHashMap()
|
||||
err = m.Load(list.GetDB("abc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Add("abc")
|
||||
|
||||
for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
|
||||
t.Log(hash, "=>", m.Exist(hash))
|
||||
}
|
||||
}
|
||||
|
||||
func Benchmark_BigInt(b *testing.B) {
|
||||
var hash = stringutil.Md5("123456")
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
var bigInt = big.NewInt(0)
|
||||
bigInt.SetString(hash, 16)
|
||||
_ = bigInt.Uint64()
|
||||
}
|
||||
}
|
||||
@@ -69,8 +69,8 @@ func TestFileList_Add_Many(t *testing.T) {
|
||||
_ = list.Close()
|
||||
}()
|
||||
|
||||
before := time.Now()
|
||||
for i := 0; i < 100_000; i++ {
|
||||
var before = time.Now()
|
||||
for i := 0; i < 10_000_000; i++ {
|
||||
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
|
||||
_ = list.Add(stringutil.Md5(u), &caches.Item{
|
||||
Key: u,
|
||||
@@ -290,12 +290,12 @@ func TestFileList_Stat(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileList_Count(t *testing.T) {
|
||||
list := caches.NewFileList(Tea.Root + "/data")
|
||||
var list = caches.NewFileList(Tea.Root + "/data")
|
||||
err := list.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := time.Now()
|
||||
var before = time.Now()
|
||||
count, err := list.Count()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -367,6 +367,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strconv"
|
||||
"sync"
|
||||
@@ -16,7 +15,7 @@ var SharedManager = NewManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventQuit, func() {
|
||||
logs.Println("CACHE", "quiting cache manager")
|
||||
remotelogs.Println("CACHE", "quiting cache manager")
|
||||
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
|
||||
})
|
||||
}
|
||||
|
||||
117
internal/conns/map.go
Normal file
117
internal/conns/map.go
Normal file
@@ -0,0 +1,117 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package conns
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var SharedMap = NewMap()
|
||||
|
||||
type Map struct {
|
||||
m map[string]map[int]net.Conn // ip => { port => Conn }
|
||||
|
||||
locker sync.RWMutex
|
||||
}
|
||||
|
||||
func NewMap() *Map {
|
||||
return &Map{
|
||||
m: map[string]map[int]net.Conn{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) Add(conn net.Conn) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var ip = tcpAddr.IP.String()
|
||||
var port = tcpAddr.Port
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
connMap, ok := this.m[ip]
|
||||
if !ok {
|
||||
this.m[ip] = map[int]net.Conn{
|
||||
port: conn,
|
||||
}
|
||||
} else {
|
||||
connMap[port] = conn
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) Remove(conn net.Conn) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var ip = tcpAddr.IP.String()
|
||||
var port = tcpAddr.Port
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
connMap, ok := this.m[ip]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(connMap, port)
|
||||
|
||||
if len(connMap) == 0 {
|
||||
delete(this.m, ip)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) CountIPConns(ip string) int {
|
||||
this.locker.RLock()
|
||||
var l = len(this.m[ip])
|
||||
this.locker.RUnlock()
|
||||
return l
|
||||
}
|
||||
|
||||
func (this *Map) CloseIPConns(ip string) {
|
||||
var conns = []net.Conn{}
|
||||
|
||||
this.locker.RLock()
|
||||
connMap, ok := this.m[ip]
|
||||
|
||||
// 复制,防止在Close时产生并发冲突
|
||||
if ok {
|
||||
for _, conn := range connMap {
|
||||
conns = append(conns, conn)
|
||||
}
|
||||
}
|
||||
|
||||
// 需要在Close之前结束,防止死循环
|
||||
this.locker.RUnlock()
|
||||
|
||||
if ok {
|
||||
for _, conn := range conns {
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
// 这里不需要从 m 中删除,因为关闭时会自然触发回调
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Map) AllConns() []net.Conn {
|
||||
this.locker.RLock()
|
||||
defer this.locker.RUnlock()
|
||||
|
||||
var result = []net.Conn{}
|
||||
for _, m := range this.m {
|
||||
for _, conn := range m {
|
||||
result = append(result, conn)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "0.5.0"
|
||||
Version = "0.5.2"
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
@@ -195,14 +195,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// new connections rate
|
||||
var newConnectionsRate = tcpConfig.NewConnectionsRate
|
||||
if newConnectionsRate <= 0 {
|
||||
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
|
||||
if newConnectionsRate <= 0 {
|
||||
newConnectionsRate = 100000
|
||||
// new connections rate (minutely)
|
||||
var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
|
||||
if newConnectionsMinutelyRate <= 0 {
|
||||
newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
|
||||
if newConnectionsMinutelyRate <= 0 {
|
||||
newConnectionsMinutelyRate = 100000
|
||||
}
|
||||
}
|
||||
var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
|
||||
if newConnectionsMinutelyRateBlockTimeout < 0 {
|
||||
newConnectionsMinutelyRateBlockTimeout = 0
|
||||
}
|
||||
|
||||
// new connections rate (secondly)
|
||||
var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
|
||||
if newConnectionsSecondlyRate <= 0 {
|
||||
newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
|
||||
if newConnectionsSecondlyRate <= 0 {
|
||||
newConnectionsSecondlyRate = 10000
|
||||
}
|
||||
}
|
||||
var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
|
||||
if newConnectionsSecondlyRateBlockTimeout < 0 {
|
||||
newConnectionsSecondlyRateBlockTimeout = 0
|
||||
}
|
||||
|
||||
// 检查是否有变化
|
||||
var hasChanges = false
|
||||
@@ -215,7 +232,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
hasChanges = true
|
||||
break
|
||||
}
|
||||
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
|
||||
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
|
||||
hasChanges = true
|
||||
break
|
||||
}
|
||||
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
|
||||
hasChanges = true
|
||||
break
|
||||
}
|
||||
@@ -251,6 +272,7 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO 让用户选择是drop还是reject
|
||||
if maxConnectionsPerIP > 0 {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
@@ -261,14 +283,47 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
|
||||
}
|
||||
}
|
||||
|
||||
if newConnectionsRate > 0 {
|
||||
// TODO 思考是否有惩罚机制
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
// 超过一定速率就drop或者加入黑名单(分钟)
|
||||
// TODO 让用户选择是drop还是reject
|
||||
if newConnectionsMinutelyRate > 0 {
|
||||
if newConnectionsMinutelyRateBlockTimeout > 0 {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
} else {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 超过一定速率就drop或者加入黑名单(秒)
|
||||
// TODO 让用户选择是drop还是reject
|
||||
if newConnectionsSecondlyRate > 0 {
|
||||
if newConnectionsSecondlyRateBlockTimeout > 0 {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
} else {
|
||||
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -335,14 +390,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
|
||||
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
|
||||
for _, rule := range rules {
|
||||
var pieces = this.decodeUserData(rule.UserData())
|
||||
if len(pieces) != 4 {
|
||||
if len(pieces) < 4 {
|
||||
continue
|
||||
}
|
||||
if pieces[0] != "tcp" {
|
||||
continue
|
||||
}
|
||||
switch pieces[2] {
|
||||
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
|
||||
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
|
||||
err := chain.DeleteRule(rule)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
47
internal/firewalls/firewall_base.go
Normal file
47
internal/firewalls/firewall_base.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BaseFirewall struct {
|
||||
locker sync.Mutex
|
||||
latestIPTimes []string // [ip@time, ....]
|
||||
}
|
||||
|
||||
// 检查是否在最近添加过
|
||||
func (this *BaseFirewall) checkLatestIP(ip string) bool {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var expiredIndex = -1
|
||||
for index, ipTime := range this.latestIPTimes {
|
||||
var pieces = strings.Split(ipTime, "@")
|
||||
var oldIP = pieces[0]
|
||||
var oldTimestamp = pieces[1]
|
||||
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
|
||||
expiredIndex = index
|
||||
continue
|
||||
}
|
||||
if oldIP == ip {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if expiredIndex > -1 {
|
||||
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
|
||||
}
|
||||
|
||||
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
|
||||
const maxLen = 128
|
||||
if len(this.latestIPTimes) > maxLen {
|
||||
this.latestIPTimes = this.latestIPTimes[1:]
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package firewalls
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
@@ -11,15 +12,22 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type firewalldCmd struct {
|
||||
cmd *exec.Cmd
|
||||
denyIP string
|
||||
}
|
||||
|
||||
type Firewalld struct {
|
||||
BaseFirewall
|
||||
|
||||
isReady bool
|
||||
exe string
|
||||
cmdQueue chan *exec.Cmd
|
||||
cmdQueue chan *firewalldCmd
|
||||
}
|
||||
|
||||
func NewFirewalld() *Firewalld {
|
||||
var firewalld = &Firewalld{
|
||||
cmdQueue: make(chan *exec.Cmd, 4096),
|
||||
cmdQueue: make(chan *firewalldCmd, 4096),
|
||||
}
|
||||
|
||||
path, err := exec.LookPath("firewall-cmd")
|
||||
@@ -41,13 +49,19 @@ func NewFirewalld() *Firewalld {
|
||||
|
||||
func (this *Firewalld) init() {
|
||||
goman.New(func() {
|
||||
for cmd := range this.cmdQueue {
|
||||
for c := range this.cmdQueue {
|
||||
var cmd = c.cmd
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
if strings.HasPrefix(err.Error(), "Warning:") {
|
||||
continue
|
||||
}
|
||||
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
|
||||
} else {
|
||||
// 关闭连接
|
||||
if len(c.denyIP) > 0 {
|
||||
conns.SharedMap.CloseIPConns(c.denyIP)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -72,7 +86,7 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
|
||||
return nil
|
||||
}
|
||||
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -82,12 +96,12 @@ func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--add-port="+port, "--permanent")
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--add-port="+port)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,7 +113,7 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
|
||||
return nil
|
||||
}
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -108,12 +122,12 @@ func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol str
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+port, "--permanent")
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
|
||||
{
|
||||
var cmd = exec.Command(this.exe, "--remove-port="+port)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -131,6 +145,12 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 避免短时间内重复添加
|
||||
if this.checkLatestIP(ip) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
@@ -140,7 +160,7 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
|
||||
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
|
||||
}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -148,6 +168,12 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) e
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 避免短时间内重复添加
|
||||
if async && this.checkLatestIP(ip) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
@@ -158,10 +184,13 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) e
|
||||
}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
if async {
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 关闭连接
|
||||
defer conns.SharedMap.CloseIPConns(ip)
|
||||
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
|
||||
@@ -173,6 +202,7 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
|
||||
if !this.isReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
var family = "ipv4"
|
||||
if strings.Contains(ip, ":") {
|
||||
family = "ipv6"
|
||||
@@ -180,14 +210,14 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
|
||||
for _, action := range []string{"reject", "drop"} {
|
||||
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
|
||||
var cmd = exec.Command(this.exe, args...)
|
||||
this.pushCmd(cmd)
|
||||
this.pushCmd(cmd, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
|
||||
func (this *Firewalld) pushCmd(cmd *exec.Cmd, denyIP string) {
|
||||
select {
|
||||
case this.cmdQueue <- cmd:
|
||||
case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
|
||||
default:
|
||||
// we discard the command
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ package firewalls
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
|
||||
@@ -100,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
|
||||
}
|
||||
|
||||
type NFTablesFirewall struct {
|
||||
BaseFirewall
|
||||
|
||||
conn *nftables.Conn
|
||||
isReady bool
|
||||
version string
|
||||
@@ -344,6 +347,14 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
|
||||
return errors.New("invalid ip '" + ip + "'")
|
||||
}
|
||||
|
||||
// 尝试关闭连接
|
||||
conns.SharedMap.CloseIPConns(ip)
|
||||
|
||||
// 避免短时间内重复添加
|
||||
if async && this.checkLatestIP(ip) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if async {
|
||||
select {
|
||||
case this.dropIPQueue <- &blockIPItem{
|
||||
@@ -357,6 +368,9 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
|
||||
return nil
|
||||
}
|
||||
|
||||
// 再次尝试关闭连接
|
||||
defer conns.SharedMap.CloseIPConns(ip)
|
||||
|
||||
if strings.Contains(ip, ":") { // ipv6
|
||||
if this.denyIPv6Set == nil {
|
||||
return errors.New("ipv6 ip set is nil")
|
||||
@@ -433,3 +447,35 @@ func (this *NFTablesFirewall) readVersion(nftPath string) string {
|
||||
}
|
||||
return versionMatches[1]
|
||||
}
|
||||
|
||||
// 检查是否在最近添加过
|
||||
func (this *NFTablesFirewall) existLatestIP(ip string) bool {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var expiredIndex = -1
|
||||
for index, ipTime := range this.latestIPTimes {
|
||||
var pieces = strings.Split(ipTime, "@")
|
||||
var oldIP = pieces[0]
|
||||
var oldTimestamp = pieces[1]
|
||||
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
|
||||
expiredIndex = index
|
||||
continue
|
||||
}
|
||||
if oldIP == ip {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if expiredIndex > -1 {
|
||||
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
|
||||
}
|
||||
|
||||
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
|
||||
const maxLen = 128
|
||||
if len(this.latestIPTimes) > maxLen {
|
||||
this.latestIPTimes = this.latestIPTimes[1:]
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
// 源码改自:https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
|
||||
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
IndexBlockLength = 12
|
||||
)
|
||||
|
||||
var err error
|
||||
|
||||
type IP2Region struct {
|
||||
headerSip []int64
|
||||
headerPtr []int64
|
||||
headerLen int64
|
||||
|
||||
// super block index info
|
||||
firstIndexPtr int64
|
||||
lastIndexPtr int64
|
||||
totalBlocks int64
|
||||
|
||||
dbData []byte
|
||||
}
|
||||
|
||||
type IpInfo struct {
|
||||
CityId int64
|
||||
Country string
|
||||
Region string
|
||||
Province string
|
||||
City string
|
||||
ISP string
|
||||
}
|
||||
|
||||
func (ip IpInfo) String() string {
|
||||
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
|
||||
}
|
||||
|
||||
func getIpInfo(cityId int64, line []byte) *IpInfo {
|
||||
lineSlice := strings.Split(string(line), "|")
|
||||
ipInfo := &IpInfo{}
|
||||
length := len(lineSlice)
|
||||
ipInfo.CityId = cityId
|
||||
if length < 5 {
|
||||
for i := 0; i <= 5-length; i++ {
|
||||
lineSlice = append(lineSlice, "")
|
||||
}
|
||||
}
|
||||
|
||||
ipInfo.Country = lineSlice[0]
|
||||
ipInfo.Region = lineSlice[1]
|
||||
ipInfo.Province = lineSlice[2]
|
||||
ipInfo.City = lineSlice[3]
|
||||
ipInfo.ISP = lineSlice[4]
|
||||
return ipInfo
|
||||
}
|
||||
|
||||
func NewIP2Region(path string) (*IP2Region, error) {
|
||||
var region = &IP2Region{}
|
||||
region.dbData, err = os.ReadFile(path)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
region.firstIndexPtr = region.ipLongAtOffset(0)
|
||||
region.lastIndexPtr = region.ipLongAtOffset(4)
|
||||
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
|
||||
return region, nil
|
||||
}
|
||||
|
||||
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
|
||||
ip, err := ip2long(ipStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
h := this.totalBlocks
|
||||
var dataPtr, l int64
|
||||
for l <= h {
|
||||
m := (l + h) >> 1
|
||||
p := this.firstIndexPtr + m*IndexBlockLength
|
||||
sip := this.ipLongAtOffset(p)
|
||||
if ip < sip {
|
||||
h = m - 1
|
||||
} else {
|
||||
eip := this.ipLongAtOffset(p + 4)
|
||||
if ip > eip {
|
||||
l = m + 1
|
||||
} else {
|
||||
dataPtr = this.ipLongAtOffset(p + 8)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if dataPtr == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
dataLen := (dataPtr >> 24) & 0xFF
|
||||
dataPtr = dataPtr & 0x00FFFFFF
|
||||
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
|
||||
}
|
||||
|
||||
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
|
||||
return int64(this.dbData[offset]) |
|
||||
int64(this.dbData[offset+1])<<8 |
|
||||
int64(this.dbData[offset+2])<<16 |
|
||||
int64(this.dbData[offset+3])<<24
|
||||
}
|
||||
|
||||
func ip2long(IpStr string) (int64, error) {
|
||||
bits := strings.Split(IpStr, ".")
|
||||
if len(bits) != 4 {
|
||||
return 0, errors.New("ip format error")
|
||||
}
|
||||
|
||||
var sum int64
|
||||
for i, n := range bits {
|
||||
bit, _ := strconv.ParseInt(n, 10, 64)
|
||||
sum += bit << uint(24-8*i)
|
||||
}
|
||||
|
||||
return sum, nil
|
||||
}
|
||||
@@ -18,7 +18,8 @@ import (
|
||||
type IPListDB struct {
|
||||
db *sql.DB
|
||||
|
||||
itemTableName string
|
||||
itemTableName string
|
||||
|
||||
deleteExpiredItemsStmt *sql.Stmt
|
||||
deleteItemStmt *sql.Stmt
|
||||
insertItemStmt *sql.Stmt
|
||||
@@ -53,7 +54,11 @@ func (this *IPListDB) init() error {
|
||||
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
var path = this.dir + "/ip_list.db"
|
||||
_ = os.Remove(path + "-shm")
|
||||
_ = os.Remove(path + "-wal")
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -194,12 +199,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
row := this.selectMaxVersionStmt.QueryRow()
|
||||
var row = this.selectMaxVersionStmt.QueryRow()
|
||||
if row == nil {
|
||||
return 0
|
||||
}
|
||||
var version int64
|
||||
err = row.Scan(&version)
|
||||
err := row.Scan(&version)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = db.AddItem(&pb.IPItem{
|
||||
Id: 1,
|
||||
IpFrom: "192.168.1.101",
|
||||
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = db.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
type LibraryInterface interface {
|
||||
// Load 加载数据库文件
|
||||
Load(dbPath string) error
|
||||
|
||||
// Lookup 查询IP
|
||||
// 返回结果有可能为空
|
||||
Lookup(ip string) (*Result, error)
|
||||
|
||||
// Close 关闭数据库文件
|
||||
Close()
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type IP2RegionLibrary struct {
|
||||
db *IP2Region
|
||||
}
|
||||
|
||||
func (this *IP2RegionLibrary) Load(dbPath string) error {
|
||||
db, err := NewIP2Region(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
this.db = db
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
|
||||
// 暂不支持IPv6
|
||||
if strings.Contains(ip, ":") {
|
||||
return nil, nil
|
||||
}
|
||||
if net.ParseIP(ip) == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if this.db == nil {
|
||||
return nil, errors.New("library has not been loaded")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// 防止panic发生
|
||||
err := recover()
|
||||
if err != nil {
|
||||
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
|
||||
}
|
||||
}()
|
||||
|
||||
info, err := this.db.MemorySearch(ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if info.Country == "0" {
|
||||
info.Country = ""
|
||||
}
|
||||
if info.Region == "0" {
|
||||
info.Region = ""
|
||||
}
|
||||
if info.Province == "0" {
|
||||
info.Province = ""
|
||||
}
|
||||
if info.City == "0" {
|
||||
info.City = ""
|
||||
}
|
||||
if info.ISP == "0" {
|
||||
info.ISP = ""
|
||||
}
|
||||
|
||||
return &Result{
|
||||
CityId: info.CityId,
|
||||
Country: info.Country,
|
||||
Region: info.Region,
|
||||
Province: info.Province,
|
||||
City: info.City,
|
||||
ISP: info.ISP,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (this *IP2RegionLibrary) Close() {
|
||||
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
|
||||
var mem = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(mem)
|
||||
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var mem2 = &runtime.MemStats{}
|
||||
runtime.ReadMemStats(mem2)
|
||||
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, ip := range []string{"8.8.9.9"} {
|
||||
result, err := library.Lookup(ip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("IP:", ip, "result:", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Lookup(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
|
||||
result, err := library.Lookup(ip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("IP:", ip, "result:", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var count = 4000
|
||||
var wg = sync.WaitGroup{}
|
||||
wg.Add(count)
|
||||
for i := 0; i < count; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
t.Log("ok")
|
||||
}
|
||||
|
||||
func TestIP2RegionLibrary_Memory(t *testing.T) {
|
||||
library := &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
before := time.Now()
|
||||
|
||||
for i := 0; i < 1_000_000; i++ {
|
||||
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
|
||||
}
|
||||
|
||||
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
|
||||
}
|
||||
|
||||
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var library = &IP2RegionLibrary{}
|
||||
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, _ = library.Lookup("8.8.8.8")
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/files"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var SharedManager = NewManager()
|
||||
var SharedLibrary LibraryInterface
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
// 初始化
|
||||
library, err := SharedManager.Load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("IP_LIBRARY", err)
|
||||
return
|
||||
}
|
||||
SharedLibrary = library
|
||||
})
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
code string
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
return &Manager{}
|
||||
}
|
||||
|
||||
func (this *Manager) Load() (LibraryInterface, error) {
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config := nodeConfig.GlobalConfig
|
||||
if config == nil {
|
||||
config = &serverconfigs.GlobalConfig{}
|
||||
}
|
||||
|
||||
// 当前正在使用的IP库代号
|
||||
code := config.IPLibrary.Code
|
||||
if len(code) == 0 {
|
||||
code = serverconfigs.DefaultIPLibraryType
|
||||
}
|
||||
|
||||
dir := Tea.Root + "/resources/ipdata/" + code
|
||||
var lastVersion int64 = -1
|
||||
lastFilename := ""
|
||||
for _, file := range files.NewFile(dir).List() {
|
||||
filename := file.Name()
|
||||
|
||||
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
|
||||
if reg.MatchString(filename) { // 先查找有版本号的
|
||||
result := reg.FindStringSubmatch(filename)
|
||||
version := types.Int64(result[1])
|
||||
if version > lastVersion {
|
||||
lastVersion = version
|
||||
lastFilename = filename
|
||||
}
|
||||
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
|
||||
if lastVersion == -1 {
|
||||
lastFilename = filename
|
||||
lastVersion = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(lastFilename) == 0 {
|
||||
return nil, errors.New("ip library file not found")
|
||||
}
|
||||
|
||||
var libraryPtr LibraryInterface
|
||||
switch code {
|
||||
case serverconfigs.IPLibraryTypeIP2Region:
|
||||
libraryPtr = &IP2RegionLibrary{}
|
||||
default:
|
||||
return nil, errors.New("invalid ip library code '" + code + "'")
|
||||
}
|
||||
|
||||
err = libraryPtr.Load(dir + "/" + lastFilename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return libraryPtr, nil
|
||||
}
|
||||
@@ -1,155 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedCityManager = NewCityManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedCityManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedCityManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// CityManager 中国省份信息管理
|
||||
type CityManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
cityMap map[string]int64 // provinceName_cityName => cityName
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewCityManager() *CityManager {
|
||||
return &CityManager{
|
||||
cacheFile: Tea.Root + "/configs/region_city.json.cache",
|
||||
cityMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("CITY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("City_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("CITY_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
|
||||
this.locker.RLock()
|
||||
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *CityManager) load() error {
|
||||
data, err := os.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.cityMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新城市信息
|
||||
func (this *CityManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionCityRPC().FindAllRegionCities(rpcClient.Context(), &pb.FindAllRegionCitiesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, city := range resp.RegionCities {
|
||||
for _, code := range city.Codes {
|
||||
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.cityMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = os.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package iplibrary
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewCityManager(t *testing.T) {
|
||||
var manager = NewCityManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup(16, "许昌市"))
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedCountryManager = NewCountryManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedCountryManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedCountryManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// CountryManager 国家/地区信息管理
|
||||
type CountryManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
countryMap map[string]int64 // countryName => countryId
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewCountryManager() *CountryManager {
|
||||
return &CountryManager{
|
||||
cacheFile: Tea.Root + "/configs/region_country.json.cache",
|
||||
countryMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
|
||||
this.locker.RLock()
|
||||
countryId, _ = this.countryMap[countryName]
|
||||
this.locker.RUnlock()
|
||||
return countryId
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *CountryManager) load() error {
|
||||
data, err := os.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.countryMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新国家信息
|
||||
func (this *CountryManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionCountryRPC().FindAllRegionCountries(rpcClient.Context(), &pb.FindAllRegionCountriesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, country := range resp.RegionCountries {
|
||||
for _, code := range country.Codes {
|
||||
m[code] = country.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.countryMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
err = os.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCountryManager_load(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.countryMap)
|
||||
}
|
||||
|
||||
func TestCountryManager_loop(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.countryMap)
|
||||
}
|
||||
|
||||
func TestCountryManager_loop_skip(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
for i := 0; i < 10; i++ {
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountryManager_Lookup(t *testing.T) {
|
||||
manager := NewCountryManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
|
||||
}
|
||||
|
||||
func BenchmarkCountryManager_Lookup(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
manager := NewCountryManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = manager.Lookup("中国")
|
||||
}
|
||||
}
|
||||
@@ -43,7 +43,7 @@ type IPListManager struct {
|
||||
|
||||
func NewIPListManager() *IPListManager {
|
||||
return &IPListManager{
|
||||
pageSize: 500,
|
||||
pageSize: 1000,
|
||||
listMap: map[int64]*IPList{},
|
||||
}
|
||||
}
|
||||
@@ -111,15 +111,19 @@ func (this *IPListManager) init() {
|
||||
var size int64 = 1000
|
||||
for {
|
||||
items, err := db.ReadItems(offset, size)
|
||||
var l = len(items)
|
||||
if err != nil {
|
||||
remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
|
||||
} else {
|
||||
if len(items) == 0 {
|
||||
if l == 0 {
|
||||
break
|
||||
}
|
||||
this.processItems(items, false)
|
||||
if int64(l) < size {
|
||||
break
|
||||
}
|
||||
}
|
||||
offset += int64(len(items))
|
||||
offset += int64(l)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -144,7 +148,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
|
||||
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
|
||||
Version: this.version,
|
||||
Size: this.pageSize,
|
||||
})
|
||||
@@ -155,7 +159,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
items := itemsResp.IpItems
|
||||
var items = itemsResp.IpItems
|
||||
if len(items) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
@@ -183,7 +187,6 @@ func (this *IPListManager) FindList(listId int64) *IPList {
|
||||
}
|
||||
|
||||
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
|
||||
this.locker.Lock()
|
||||
var changedLists = map[*IPList]zero.Zero{}
|
||||
for _, item := range items {
|
||||
var list *IPList
|
||||
@@ -203,11 +206,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
|
||||
list = GlobalWhiteIPList
|
||||
}
|
||||
} else { // 其他List
|
||||
this.locker.Lock()
|
||||
list = this.listMap[item.ListId]
|
||||
this.locker.Unlock()
|
||||
}
|
||||
if list == nil {
|
||||
list = NewIPList()
|
||||
this.locker.Lock()
|
||||
this.listMap[item.ListId] = list
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
changedLists[list] = zero.New()
|
||||
@@ -246,8 +253,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
|
||||
changedList.Sort()
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
|
||||
if fromRemote {
|
||||
var latestVersion = items[len(items)-1].Version
|
||||
if latestVersion > this.version {
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedProviderManager = NewProviderManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedProviderManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedProviderManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// ProviderManager 中国省份信息管理
|
||||
type ProviderManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
providerMap map[string]int64 // name => id
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewProviderManager() *ProviderManager {
|
||||
return &ProviderManager{
|
||||
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
|
||||
providerMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
|
||||
this.locker.RLock()
|
||||
providerId, _ = this.providerMap[providerName]
|
||||
this.locker.RUnlock()
|
||||
return
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *ProviderManager) load() error {
|
||||
data, err := os.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.providerMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新服务商信息
|
||||
func (this *ProviderManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionProviderRPC().FindAllRegionProviders(rpcClient.Context(), &pb.FindAllRegionProvidersRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, provider := range resp.RegionProviders {
|
||||
for _, code := range provider.Codes {
|
||||
m[code] = provider.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.providerMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = os.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
|
||||
package iplibrary
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewProviderManager(t *testing.T) {
|
||||
var manager = NewProviderManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("阿里云"))
|
||||
t.Log(manager.Lookup("阿里云2"))
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ChinaCountryId int64 = 1
|
||||
)
|
||||
|
||||
var SharedProvinceManager = NewProvinceManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
SharedProvinceManager.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedProvinceManager.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// ProvinceManager 中国省份信息管理
|
||||
type ProvinceManager struct {
|
||||
ticker *time.Ticker
|
||||
|
||||
cacheFile string
|
||||
|
||||
provinceMap map[string]int64 // provinceName => provinceId
|
||||
dataHash string // 国家JSON的md5
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
isUpdated bool
|
||||
}
|
||||
|
||||
func NewProvinceManager() *ProvinceManager {
|
||||
return &ProvinceManager{
|
||||
cacheFile: Tea.Root + "/configs/region_province.json.cache",
|
||||
provinceMap: map[string]int64{},
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Start() {
|
||||
// 从缓存中读取
|
||||
err := this.load()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
}
|
||||
|
||||
// 第一次更新
|
||||
err = this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
}
|
||||
|
||||
// 定时更新
|
||||
this.ticker = time.NewTicker(4 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
|
||||
this.locker.RLock()
|
||||
provinceId, _ = this.provinceMap[provinceName]
|
||||
this.locker.RUnlock()
|
||||
return provinceId
|
||||
}
|
||||
|
||||
// 从缓存中读取
|
||||
func (this *ProvinceManager) load() error {
|
||||
data, err := os.ReadFile(this.cacheFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
m := map[string]int64{}
|
||||
err = json.Unmarshal(data, &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m != nil && len(m) > 0 {
|
||||
this.provinceMap = m
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新省份信息
|
||||
func (this *ProvinceManager) loop() error {
|
||||
if this.isUpdated {
|
||||
return nil
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.RegionProvinceRPC().FindAllRegionProvincesWithRegionCountryId(rpcClient.Context(), &pb.FindAllRegionProvincesWithRegionCountryIdRequest{
|
||||
RegionCountryId: ChinaCountryId,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m := map[string]int64{}
|
||||
for _, province := range resp.RegionProvinces {
|
||||
for _, code := range province.Codes {
|
||||
m[code] = province.Id
|
||||
}
|
||||
}
|
||||
|
||||
// 检查是否有更新
|
||||
data, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
if this.dataHash == dataHash {
|
||||
return nil
|
||||
}
|
||||
this.dataHash = dataHash
|
||||
|
||||
this.locker.Lock()
|
||||
this.provinceMap = m
|
||||
this.isUpdated = true
|
||||
this.locker.Unlock()
|
||||
|
||||
// 保存到本地缓存
|
||||
|
||||
err = os.WriteFile(this.cacheFile, data, 0666)
|
||||
return err
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProvinceManager_load(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.provinceMap)
|
||||
}
|
||||
|
||||
func TestProvinceManager_loop(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok", manager.provinceMap)
|
||||
}
|
||||
|
||||
func TestProvinceManager_loop_skip(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
for i := 0; i < 10; i++ {
|
||||
err := manager.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProvinceManager_Lookup(t *testing.T) {
|
||||
manager := NewProvinceManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
|
||||
}
|
||||
|
||||
func BenchmarkProvinceManager_Lookup(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
manager := NewProvinceManager()
|
||||
err := manager.load()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = manager.Lookup("安徽省")
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManager_Load(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
manager := NewManager()
|
||||
lib, err := manager.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(lib.Lookup("1.2.3.4"))
|
||||
t.Log(lib.Lookup("2.3.4.5"))
|
||||
t.Log(lib.Lookup("200.200.200.200"))
|
||||
t.Log(lib.Lookup("202.106.0.20"))
|
||||
}
|
||||
|
||||
func TestNewManager(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
t.Log(SharedLibrary)
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedUpdater = NewUpdater()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventStart, func() {
|
||||
goman.New(func() {
|
||||
SharedUpdater.Start()
|
||||
})
|
||||
})
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedUpdater.Stop()
|
||||
})
|
||||
}
|
||||
|
||||
// Updater IP库更新程序
|
||||
type Updater struct {
|
||||
ticker *time.Ticker
|
||||
}
|
||||
|
||||
// NewUpdater 获取新对象
|
||||
func NewUpdater() *Updater {
|
||||
return &Updater{}
|
||||
}
|
||||
|
||||
// Start 开始更新
|
||||
func (this *Updater) Start() {
|
||||
// 这里不需要太频繁检查更新,因为通常不需要更新IP库
|
||||
this.ticker = time.NewTicker(1 * time.Hour)
|
||||
for range this.ticker.C {
|
||||
err := this.loop()
|
||||
if err != nil {
|
||||
remotelogs.ErrorObject("IP_LIBRARY", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Updater) Stop() {
|
||||
if this.ticker != nil {
|
||||
this.ticker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// 单次任务
|
||||
func (this *Updater) loop() error {
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if nodeConfig.GlobalConfig == nil {
|
||||
return nil
|
||||
}
|
||||
code := nodeConfig.GlobalConfig.IPLibrary.Code
|
||||
if len(code) == 0 {
|
||||
code = serverconfigs.DefaultIPLibraryType
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lib := libraryResp.IpLibrary
|
||||
if lib == nil || lib.File == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
typeInfo := serverconfigs.FindIPLibraryWithType(code)
|
||||
if typeInfo == nil {
|
||||
return errors.New("invalid ip library code '" + code + "'")
|
||||
}
|
||||
|
||||
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
|
||||
|
||||
// 是否已经存在
|
||||
_, err = os.Stat(path)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 开始下载
|
||||
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunkIds := fileChunkIdsResp.FileChunkIds
|
||||
if len(chunkIds) == 0 {
|
||||
return nil
|
||||
}
|
||||
isOk := false
|
||||
|
||||
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
// 如果保存不成功就直接删除
|
||||
if !isOk {
|
||||
_ = fp.Close()
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
}()
|
||||
for _, chunkId := range chunkIds {
|
||||
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunk := chunkResp.FileChunk
|
||||
|
||||
if chunk == nil {
|
||||
continue
|
||||
}
|
||||
_, err = fp.Write(chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = fp.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 重新加载
|
||||
library, err := SharedManager.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
SharedLibrary = library
|
||||
|
||||
isOk = true
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package iplibrary
|
||||
|
||||
import (
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/dbs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpdater_loop(t *testing.T) {
|
||||
dbs.NotifyReady()
|
||||
|
||||
updater := NewUpdater()
|
||||
err := updater.loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("ok")
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package metrics
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"strconv"
|
||||
"sync"
|
||||
@@ -11,7 +12,15 @@ import (
|
||||
|
||||
var SharedManager = NewManager()
|
||||
|
||||
func init() {
|
||||
events.On(events.EventQuit, func() {
|
||||
SharedManager.Quit()
|
||||
})
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
isQuiting bool
|
||||
|
||||
tasks map[int64]*Task // itemId => *Task
|
||||
categoryTasks map[string][]*Task // category => []*Task
|
||||
locker sync.RWMutex
|
||||
@@ -29,6 +38,10 @@ func NewManager() *Manager {
|
||||
}
|
||||
|
||||
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
|
||||
if this.isQuiting {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
@@ -101,6 +114,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
|
||||
|
||||
// Add 添加数据
|
||||
func (this *Manager) Add(obj MetricInterface) {
|
||||
if this.isQuiting {
|
||||
return
|
||||
}
|
||||
|
||||
this.locker.RLock()
|
||||
for _, task := range this.categoryTasks[obj.MetricCategory()] {
|
||||
task.Add(obj)
|
||||
@@ -119,3 +136,17 @@ func (this *Manager) HasTCPMetrics() bool {
|
||||
func (this *Manager) HasUDPMetrics() bool {
|
||||
return this.hasUDPMetrics
|
||||
}
|
||||
|
||||
// Quit 退出管理器
|
||||
func (this *Manager) Quit() {
|
||||
this.isQuiting = true
|
||||
|
||||
remotelogs.Println("METRIC_MANAGER", "quit")
|
||||
|
||||
this.locker.Lock()
|
||||
for _, task := range this.tasks {
|
||||
_ = task.Stop()
|
||||
}
|
||||
this.tasks = map[int64]*Task{}
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"os"
|
||||
"strconv"
|
||||
@@ -89,7 +90,11 @@ func (this *Task) Init() error {
|
||||
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
|
||||
_ = os.Remove(path + "-shm")
|
||||
_ = os.Remove(path + "-wal")
|
||||
|
||||
db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -424,7 +429,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
|
||||
_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
|
||||
MetricStats: pbStats,
|
||||
Time: currentTime,
|
||||
ServerId: serverId,
|
||||
|
||||
@@ -69,7 +69,7 @@ func (this *ValueQueue) Loop() error {
|
||||
}
|
||||
|
||||
for value := range this.valuesChan {
|
||||
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
|
||||
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
|
||||
Item: value.Item,
|
||||
ValueJSON: value.ValueJSON,
|
||||
CreatedAt: value.CreatedAt,
|
||||
|
||||
@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
|
||||
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
|
||||
if err != nil {
|
||||
statusErr, ok:= status.FromError(err)
|
||||
statusErr, ok := status.FromError(err)
|
||||
if ok {
|
||||
logs.Println(statusErr.Code())
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func (this *APIStream) loop() error {
|
||||
cancelFunc()
|
||||
}()
|
||||
|
||||
nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
|
||||
nodeStream, err := rpcClient.NodeRPC.NodeStream(ctx)
|
||||
if err != nil {
|
||||
if this.isQuiting {
|
||||
return nil
|
||||
|
||||
@@ -5,12 +5,14 @@ package nodes
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net"
|
||||
"os"
|
||||
@@ -22,6 +24,8 @@ import (
|
||||
|
||||
// ClientConn 客户端连接
|
||||
type ClientConn struct {
|
||||
BaseClientConn
|
||||
|
||||
once sync.Once
|
||||
|
||||
isTLS bool
|
||||
@@ -31,29 +35,28 @@ type ClientConn struct {
|
||||
isLO bool // 是否为环路
|
||||
|
||||
hasResetSYNFlood bool
|
||||
|
||||
BaseClientConn
|
||||
}
|
||||
|
||||
func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
|
||||
if quickClose {
|
||||
// TCP
|
||||
tcpConn, ok := conn.(*net.TCPConn)
|
||||
if ok {
|
||||
// TODO 可以在配置中设置此值
|
||||
_ = tcpConn.SetLinger(nodeconfigs.DefaultTCPLinger)
|
||||
}
|
||||
}
|
||||
|
||||
func NewClientConn(rawConn net.Conn, isTLS bool, quickClose bool) net.Conn {
|
||||
// 是否为环路
|
||||
var remoteAddr = conn.RemoteAddr().String()
|
||||
var remoteAddr = rawConn.RemoteAddr().String()
|
||||
var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
|
||||
|
||||
return &ClientConn{
|
||||
BaseClientConn: BaseClientConn{rawConn: conn},
|
||||
var conn = &ClientConn{
|
||||
BaseClientConn: BaseClientConn{rawConn: rawConn},
|
||||
isTLS: isTLS,
|
||||
isLO: isLO,
|
||||
}
|
||||
|
||||
if quickClose {
|
||||
// TODO 可以在配置中设置此值
|
||||
_ = conn.SetLinger(nodeconfigs.DefaultTCPLinger)
|
||||
}
|
||||
|
||||
// 加入到Map
|
||||
conns.SharedMap.Add(conn)
|
||||
|
||||
return conn
|
||||
}
|
||||
|
||||
func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
@@ -110,11 +113,10 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
|
||||
func (this *ClientConn) Write(b []byte) (n int, err error) {
|
||||
n, err = this.rawConn.Write(b)
|
||||
if n > 0 {
|
||||
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
|
||||
|
||||
// 统计当前服务带宽
|
||||
if this.serverId > 0 {
|
||||
if !this.isLO { // 环路不统计带宽,避免缓存预热等行为产生带宽
|
||||
if !this.isLO || Tea.IsTesting() { // 环路不统计带宽,避免缓存预热等行为产生带宽
|
||||
atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
|
||||
stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
|
||||
}
|
||||
}
|
||||
@@ -132,6 +134,9 @@ func (this *ClientConn) Close() error {
|
||||
// 不能加条件限制,因为服务配置随时有变化
|
||||
sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
|
||||
|
||||
// 从conn map中移除
|
||||
conns.SharedMap.Remove(this)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -177,6 +182,11 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
|
||||
if timeout <= 0 {
|
||||
timeout = 600
|
||||
}
|
||||
|
||||
// 关闭当前连接
|
||||
_ = this.SetLinger(0)
|
||||
_ = this.Close()
|
||||
|
||||
waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击,当前1分钟"+types.String(result)+"次空连接")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
package nodes
|
||||
|
||||
import "net"
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
)
|
||||
|
||||
type BaseClientConn struct {
|
||||
rawConn net.Conn
|
||||
@@ -42,6 +45,17 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
|
||||
// SetServerId 设置服务ID
|
||||
func (this *BaseClientConn) SetServerId(serverId int64) {
|
||||
this.serverId = serverId
|
||||
|
||||
// 设置包装前连接
|
||||
switch conn := this.rawConn.(type) {
|
||||
case *tls.Conn:
|
||||
nativeConn, ok := conn.NetConn().(ClientConnInterface)
|
||||
if ok {
|
||||
nativeConn.SetServerId(serverId)
|
||||
}
|
||||
case *ClientConn:
|
||||
conn.SetServerId(serverId)
|
||||
}
|
||||
}
|
||||
|
||||
// ServerId 读取当前连接绑定的服务ID
|
||||
@@ -52,6 +66,17 @@ func (this *BaseClientConn) ServerId() int64 {
|
||||
// SetUserId 设置所属服务的用户ID
|
||||
func (this *BaseClientConn) SetUserId(userId int64) {
|
||||
this.userId = userId
|
||||
|
||||
// 设置包装前连接
|
||||
switch conn := this.rawConn.(type) {
|
||||
case *tls.Conn:
|
||||
nativeConn, ok := conn.NetConn().(ClientConnInterface)
|
||||
if ok {
|
||||
nativeConn.SetUserId(userId)
|
||||
}
|
||||
case *ClientConn:
|
||||
conn.SetUserId(userId)
|
||||
}
|
||||
}
|
||||
|
||||
// UserId 获取当前连接所属服务的用户ID
|
||||
@@ -66,9 +91,20 @@ func (this *BaseClientConn) RawIP() string {
|
||||
}
|
||||
|
||||
// TCPConn 转换为TCPConn
|
||||
func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
|
||||
conn, ok := this.rawConn.(*net.TCPConn)
|
||||
return conn, ok
|
||||
func (this *BaseClientConn) TCPConn() (tcpConn *net.TCPConn, ok bool) {
|
||||
// 设置包装前连接
|
||||
switch conn := this.rawConn.(type) {
|
||||
case *tls.Conn:
|
||||
var internalConn = conn.NetConn()
|
||||
clientConn, ok := internalConn.(*ClientConn)
|
||||
if ok {
|
||||
return clientConn.TCPConn()
|
||||
}
|
||||
tcpConn, ok = internalConn.(*net.TCPConn)
|
||||
default:
|
||||
tcpConn, ok = this.rawConn.(*net.TCPConn)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// SetLinger 设置Linger
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"net"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClientListener 客户端网络监听
|
||||
@@ -42,10 +43,28 @@ func (this *ClientListener) Accept() (net.Conn, error) {
|
||||
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
|
||||
if err == nil {
|
||||
canGoNext, _ := iplibrary.AllowIP(ip, 0)
|
||||
var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
|
||||
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
|
||||
if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
|
||||
expiresAt, ok := waf.SharedIPBlackList.ContainsExpires(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
|
||||
if ok {
|
||||
var timeout = expiresAt - time.Now().Unix()
|
||||
if timeout > 0 {
|
||||
canGoNext = false
|
||||
|
||||
if !canGoNext || beingDenied {
|
||||
if timeout > 3600 {
|
||||
timeout = 3600
|
||||
}
|
||||
|
||||
// 使用本地防火墙延长封禁
|
||||
var fw = firewalls.Firewall()
|
||||
if fw != nil && !fw.IsMock() {
|
||||
// 这里 int(int64) 转换的前提是限制了 timeout <= 3600,否则将有整型溢出的风险
|
||||
_ = fw.DropSourceIP(ip, int(timeout), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !canGoNext {
|
||||
tcpConn, ok := conn.(*net.TCPConn)
|
||||
if ok {
|
||||
_ = tcpConn.SetLinger(0)
|
||||
@@ -53,14 +72,6 @@ func (this *ClientListener) Accept() (net.Conn, error) {
|
||||
|
||||
_ = conn.Close()
|
||||
|
||||
// 使用本地防火墙延长封禁
|
||||
if beingDenied {
|
||||
var fw = firewalls.Firewall()
|
||||
if fw != nil && !fw.IsMock() {
|
||||
_ = fw.DropSourceIP(ip, 120, true)
|
||||
}
|
||||
}
|
||||
|
||||
return this.Accept()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ Loop:
|
||||
this.rpcClient = client
|
||||
}
|
||||
|
||||
_, err := this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
_, err := this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
if err != nil {
|
||||
// 是否包含了invalid UTF-8
|
||||
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
|
||||
@@ -105,7 +105,7 @@ Loop:
|
||||
}
|
||||
|
||||
// 重新提交
|
||||
_, err = this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
|
||||
// logs.PrintAsJSON(accessLog)
|
||||
|
||||
//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
|
||||
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
accessLog,
|
||||
}})
|
||||
if err != nil {
|
||||
@@ -99,7 +99,7 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
|
||||
accessLog,
|
||||
}})
|
||||
if err != nil {
|
||||
|
||||
@@ -81,7 +81,7 @@ func (this *HTTPCacheTaskManager) Start() {
|
||||
|
||||
if rpcClient != nil {
|
||||
for taskReq := range this.taskQueue {
|
||||
_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
|
||||
_, err := rpcClient.ServerRPC.PurgeServerCache(rpcClient.Context(), taskReq)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
|
||||
}
|
||||
@@ -104,8 +104,12 @@ func (this *HTTPCacheTaskManager) Loop() error {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
|
||||
resp, err := rpcClient.HTTPCacheTaskKeyRPC.FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
|
||||
if err != nil {
|
||||
// 忽略连接错误
|
||||
if rpc.IsConnError(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -131,7 +135,7 @@ func (this *HTTPCacheTaskManager) Loop() error {
|
||||
pbResults = append(pbResults, pbResult)
|
||||
}
|
||||
|
||||
_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
|
||||
_, err = rpcClient.HTTPCacheTaskKeyRPC.UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/metrics"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
@@ -677,7 +677,7 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
case "remoteAddrValue":
|
||||
return this.requestRemoteAddr(false)
|
||||
case "rawRemoteAddr":
|
||||
addr := this.RawReq.RemoteAddr
|
||||
var addr = this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err == nil {
|
||||
addr = host
|
||||
@@ -928,42 +928,47 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
|
||||
// geo
|
||||
if prefix == "geo" {
|
||||
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
|
||||
var result = iplib.LookupIP(this.requestRemoteAddr(true))
|
||||
|
||||
switch suffix {
|
||||
case "country.name":
|
||||
if result != nil {
|
||||
return result.Country
|
||||
if result != nil && result.IsOk() {
|
||||
return result.CountryName()
|
||||
}
|
||||
return ""
|
||||
case "country.id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.CountryId())
|
||||
}
|
||||
return "0"
|
||||
case "province.name":
|
||||
if result != nil {
|
||||
return result.Province
|
||||
if result != nil && result.IsOk() {
|
||||
return result.ProvinceName()
|
||||
}
|
||||
return ""
|
||||
case "province.id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.ProvinceId())
|
||||
}
|
||||
return "0"
|
||||
case "city.name":
|
||||
if result != nil {
|
||||
return result.City
|
||||
if result != nil && result.IsOk() {
|
||||
return result.CityName()
|
||||
}
|
||||
return ""
|
||||
case "city.id":
|
||||
if result != nil {
|
||||
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
|
||||
if provinceId > 0 {
|
||||
return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
|
||||
} else {
|
||||
return "0"
|
||||
}
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.CityId())
|
||||
}
|
||||
return "0"
|
||||
case "town.name":
|
||||
if result != nil && result.IsOk() {
|
||||
return result.TownName()
|
||||
}
|
||||
return ""
|
||||
case "town.id":
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.TownId())
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
@@ -971,16 +976,16 @@ func (this *HTTPRequest) Format(source string) string {
|
||||
|
||||
// ips
|
||||
if prefix == "isp" {
|
||||
result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
|
||||
var result = iplib.LookupIP(this.requestRemoteAddr(true))
|
||||
|
||||
switch suffix {
|
||||
case "name":
|
||||
if result != nil {
|
||||
return result.ISP
|
||||
if result != nil && result.IsOk() {
|
||||
return result.ProviderName()
|
||||
}
|
||||
case "id":
|
||||
if result != nil {
|
||||
return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
|
||||
if result != nil && result.IsOk() {
|
||||
return types.String(result.ProviderId())
|
||||
}
|
||||
return "0"
|
||||
}
|
||||
@@ -1098,7 +1103,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
|
||||
}
|
||||
|
||||
// Remote-Addr
|
||||
remoteAddr := this.RawReq.RemoteAddr
|
||||
var remoteAddr = this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err == nil {
|
||||
if supportVar {
|
||||
@@ -1167,7 +1172,7 @@ func (this *HTTPRequest) requestRemoteUser() string {
|
||||
|
||||
// Path 请求的URL中路径部分
|
||||
func (this *HTTPRequest) Path() string {
|
||||
uri, err := url.ParseRequestURI(this.rawURI)
|
||||
uri, err := url.ParseRequestURI(this.uri)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -1315,7 +1320,7 @@ func (this *HTTPRequest) RemoteAddr() string {
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) RawRemoteAddr() string {
|
||||
addr := this.RawReq.RemoteAddr
|
||||
var addr = this.RawReq.RemoteAddr
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err == nil {
|
||||
addr = host
|
||||
@@ -1423,11 +1428,16 @@ func (this *HTTPRequest) Done() {
|
||||
func (this *HTTPRequest) Close() {
|
||||
this.Done()
|
||||
|
||||
requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
|
||||
if requestConn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
lingerConn, ok := requestConn.(LingerConn)
|
||||
if ok {
|
||||
_ = lingerConn.SetLinger(0)
|
||||
}
|
||||
|
||||
conn, ok := requestConn.(net.Conn)
|
||||
if ok {
|
||||
_ = conn.Close()
|
||||
@@ -1721,7 +1731,7 @@ func (this *HTTPRequest) canIgnore(err error) bool {
|
||||
}
|
||||
|
||||
// HTTP/2流错误
|
||||
if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
|
||||
if err.Error() == "http2: stream closed" || strings.Contains(err.Error(), "stream error") || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ func (this *HTTPRequest) doACME() (shouldStop bool) {
|
||||
return false
|
||||
}
|
||||
|
||||
keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
|
||||
keyResp, err := rpcClient.ACMEAuthenticationRPC.FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
|
||||
if err != nil {
|
||||
remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
|
||||
return false
|
||||
|
||||
@@ -19,7 +19,10 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
|
||||
continue
|
||||
}
|
||||
b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
|
||||
if !ref.AuthPolicy.MatchRequest(this.RawReq) {
|
||||
continue
|
||||
}
|
||||
ok, newURI, uriChanged, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
|
||||
subReq.TLS = this.RawReq.TLS
|
||||
subReq.RemoteAddr = this.RawReq.RemoteAddr
|
||||
subReq.Host = this.RawReq.Host
|
||||
@@ -36,24 +39,32 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
|
||||
return
|
||||
}
|
||||
if b {
|
||||
if ok {
|
||||
if uriChanged {
|
||||
this.uri = newURI
|
||||
}
|
||||
this.tags = append(this.tags, ref.AuthPolicy.Type)
|
||||
return
|
||||
} else {
|
||||
// Basic Auth比较特殊
|
||||
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
|
||||
var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.ReqHost
|
||||
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
if ok {
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.ReqHost
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
this.writer.WriteHeader(http.StatusUnauthorized)
|
||||
this.tags = append(this.tags, ref.AuthPolicy.Type)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,12 +44,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
// 检查服务独立的缓存条件
|
||||
refType := ""
|
||||
for _, cacheRef := range this.web.Cache.CacheRefs {
|
||||
if !cacheRef.IsOn ||
|
||||
cacheRef.Conds == nil ||
|
||||
!cacheRef.Conds.HasRequestConds() {
|
||||
if !cacheRef.IsOn {
|
||||
continue
|
||||
}
|
||||
if cacheRef.Conds.MatchRequest(this.Format) {
|
||||
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
|
||||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
|
||||
if cacheRef.IsReverse {
|
||||
return
|
||||
}
|
||||
@@ -61,12 +60,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
|
||||
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
|
||||
// 检查策略默认的缓存条件
|
||||
for _, cacheRef := range cachePolicy.CacheRefs {
|
||||
if !cacheRef.IsOn ||
|
||||
cacheRef.Conds == nil ||
|
||||
!cacheRef.Conds.HasRequestConds() {
|
||||
if !cacheRef.IsOn {
|
||||
continue
|
||||
}
|
||||
if cacheRef.Conds.MatchRequest(this.Format) {
|
||||
if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
|
||||
(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
|
||||
if cacheRef.IsReverse {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -34,17 +34,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
|
||||
}
|
||||
return this.RawReq.ContentLength + hl, true
|
||||
case "${countConnection}":
|
||||
metricNewConnMapLocker.Lock()
|
||||
_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
|
||||
if ok {
|
||||
delete(metricNewConnMap, this.RawReq.RemoteAddr)
|
||||
}
|
||||
metricNewConnMapLocker.Unlock()
|
||||
if ok {
|
||||
return 1, true
|
||||
} else {
|
||||
return 0, false
|
||||
}
|
||||
return 1, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
@@ -252,12 +252,6 @@ func (this *HTTPRequest) doOriginRequest(failedOriginIds []int64, failedLnNodeId
|
||||
return
|
||||
}
|
||||
|
||||
// 在HTTP/2下需要防止因为requestBody而导致Content-Length为空的问题
|
||||
if this.RawReq.ProtoMajor == 2 && this.RawReq.ContentLength == 0 {
|
||||
_ = this.RawReq.Body.Close()
|
||||
this.RawReq.Body = nil
|
||||
}
|
||||
|
||||
// 开始请求
|
||||
resp, err := client.Do(this.RawReq)
|
||||
if err != nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package nodes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
@@ -161,56 +162,48 @@ func (this *HTTPRequest) checkWAFRequest(firewallPolicy *firewallconfigs.HTTPFir
|
||||
|
||||
// 检查地区封禁
|
||||
if firewallPolicy.Mode == firewallconfigs.FirewallModeDefend {
|
||||
if iplibrary.SharedLibrary != nil {
|
||||
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
|
||||
regionConfig := firewallPolicy.Inbound.Region
|
||||
if regionConfig.IsNotEmpty() {
|
||||
for _, remoteAddr := range remoteAddrs {
|
||||
result, err := iplibrary.SharedLibrary.Lookup(remoteAddr)
|
||||
if err != nil {
|
||||
remotelogs.Error("HTTP_REQUEST_WAF", "iplibrary lookup failed: "+err.Error())
|
||||
} else if result != nil {
|
||||
// 检查国家级别封禁
|
||||
if len(regionConfig.DenyCountryIds) > 0 && len(result.Country) > 0 {
|
||||
countryId := iplibrary.SharedCountryManager.Lookup(result.Country)
|
||||
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
if firewallPolicy.Inbound.Region != nil && firewallPolicy.Inbound.Region.IsOn {
|
||||
regionConfig := firewallPolicy.Inbound.Region
|
||||
if regionConfig.IsNotEmpty() {
|
||||
for _, remoteAddr := range remoteAddrs {
|
||||
var result = iplib.LookupIP(remoteAddr)
|
||||
if result != nil && result.IsOk() {
|
||||
// 检查国家/地区级别封禁
|
||||
var countryId = result.CountryId()
|
||||
if countryId > 0 && lists.ContainsInt64(regionConfig.DenyCountryIds, countryId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyCountry")
|
||||
}
|
||||
|
||||
return true, false
|
||||
}
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyCountry")
|
||||
}
|
||||
|
||||
// 检查省份封禁
|
||||
if len(regionConfig.DenyProvinceIds) > 0 && len(result.Province) > 0 {
|
||||
var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
|
||||
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
return true, false
|
||||
}
|
||||
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
// 检查省份封禁
|
||||
var provinceId = result.ProvinceId()
|
||||
if provinceId > 0 && lists.ContainsInt64(regionConfig.DenyProvinceIds, provinceId) {
|
||||
this.firewallPolicyId = firewallPolicy.Id
|
||||
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyProvince")
|
||||
}
|
||||
this.writeCode(http.StatusForbidden)
|
||||
this.writer.Flush()
|
||||
this.writer.Close()
|
||||
|
||||
return true, false
|
||||
}
|
||||
// 停止日志
|
||||
if !logDenying {
|
||||
this.disableLog = true
|
||||
} else {
|
||||
this.tags = append(this.tags, "denyProvince")
|
||||
}
|
||||
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -379,6 +372,8 @@ func (this *HTTPRequest) WAFServerId() int64 {
|
||||
// WAFClose 关闭连接
|
||||
func (this *HTTPRequest) WAFClose() {
|
||||
this.Close()
|
||||
|
||||
// 这里不要强关IP所有连接,避免因为单个服务而影响所有
|
||||
}
|
||||
|
||||
func (this *HTTPRequest) WAFOnAction(action interface{}) (goNext bool) {
|
||||
|
||||
@@ -1007,30 +1007,35 @@ func (this *HTTPWriter) finishWebP() {
|
||||
Exact: true,
|
||||
})
|
||||
} else if gifImage != nil {
|
||||
anim := gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||
var anim = gowebp.NewWebpAnimation(gifImage.Config.Width, gifImage.Config.Height, gifImage.LoopCount)
|
||||
|
||||
anim.WebPAnimEncoderOptions.SetKmin(9)
|
||||
anim.WebPAnimEncoderOptions.SetKmax(17)
|
||||
defer anim.ReleaseMemory()
|
||||
webpConfig := gowebp.NewWebpConfig()
|
||||
var webpConfig = gowebp.NewWebpConfig()
|
||||
//webpConfig.SetLossless(1)
|
||||
webpConfig.SetQuality(f)
|
||||
|
||||
var timeline = 0
|
||||
|
||||
var lastErr error
|
||||
for i, img := range gifImage.Image {
|
||||
err = anim.AddFrame(img, timeline, webpConfig)
|
||||
if err != nil {
|
||||
break
|
||||
// 有错误直接跳过
|
||||
lastErr = err
|
||||
err = nil
|
||||
}
|
||||
timeline += gifImage.Delay[i] * 10
|
||||
}
|
||||
if err == nil {
|
||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||
|
||||
if err == nil {
|
||||
err = anim.Encode(this.writer)
|
||||
}
|
||||
if lastErr != nil {
|
||||
remotelogs.Error("HTTP_WRITER", "'"+this.req.URL()+"' encode webp failed: "+lastErr.Error())
|
||||
}
|
||||
err = anim.AddFrame(nil, timeline, webpConfig)
|
||||
|
||||
if err == nil {
|
||||
err = anim.Encode(this.writer)
|
||||
}
|
||||
|
||||
anim.ReleaseMemory()
|
||||
}
|
||||
|
||||
if err != nil && !this.req.canIgnore(err) {
|
||||
|
||||
103
internal/nodes/ip_library_updater.go
Normal file
103
internal/nodes/ip_library_updater.go
Normal file
@@ -0,0 +1,103 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type IPLibraryUpdater struct {
|
||||
}
|
||||
|
||||
func NewIPLibraryUpdater() *IPLibraryUpdater {
|
||||
return &IPLibraryUpdater{}
|
||||
}
|
||||
|
||||
// DataDir 文件目录
|
||||
func (this *IPLibraryUpdater) DataDir() string {
|
||||
// data/
|
||||
var dir = Tea.Root + "/data"
|
||||
stat, err := os.Stat(dir)
|
||||
if err == nil && stat.IsDir() {
|
||||
return dir
|
||||
}
|
||||
|
||||
err = os.Mkdir(dir, 0666)
|
||||
if err == nil {
|
||||
return dir
|
||||
}
|
||||
|
||||
remotelogs.Error("IP_LIBRARY_UPDATER", "create directory '"+dir+"' failed: "+err.Error())
|
||||
|
||||
// 如果不能创建 data/ 目录,那么使用临时目录
|
||||
return os.TempDir()
|
||||
}
|
||||
|
||||
// FindLatestFile 检查最新的IP库文件
|
||||
func (this *IPLibraryUpdater) FindLatestFile() (code string, fileId int64, err error) {
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
resp, err := rpcClient.IPLibraryArtifactRPC.FindPublicIPLibraryArtifact(rpcClient.Context(), &pb.FindPublicIPLibraryArtifactRequest{})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
var artifact = resp.IpLibraryArtifact
|
||||
if artifact == nil {
|
||||
return
|
||||
}
|
||||
return artifact.Code, artifact.FileId, nil
|
||||
}
|
||||
|
||||
// DownloadFile 下载文件
|
||||
func (this *IPLibraryUpdater) DownloadFile(fileId int64, writer io.Writer) error {
|
||||
if fileId <= 0 {
|
||||
return errors.New("invalid fileId: " + types.String(fileId))
|
||||
}
|
||||
|
||||
rpcClient, err := rpc.SharedRPC()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunkIdsResp, err := rpcClient.FileChunkRPC.FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: fileId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, chunkId := range chunkIdsResp.FileChunkIds {
|
||||
chunkResp, err := rpcClient.FileChunkRPC.DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var chunk = chunkResp.FileChunk
|
||||
if chunk == nil {
|
||||
return errors.New("can not find file chunk with chunk id '" + types.String(chunkId) + "'")
|
||||
}
|
||||
_, err = writer.Write(chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LogInfo 普通日志
|
||||
func (this *IPLibraryUpdater) LogInfo(message string) {
|
||||
remotelogs.Println("IP_LIBRARY_UPDATER", message)
|
||||
}
|
||||
|
||||
// LogError 错误日志
|
||||
func (this *IPLibraryUpdater) LogError(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
remotelogs.Error("IP_LIBRARY_UPDATER", err.Error())
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/tls"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"golang.org/x/net/http2"
|
||||
"io"
|
||||
@@ -13,14 +12,11 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var httpErrorLogger = log.New(io.Discard, "", 0)
|
||||
var metricNewConnMap = map[string]zero.Zero{} // remoteAddr => bool
|
||||
var metricNewConnMapLocker = &sync.Mutex{}
|
||||
|
||||
type contextKey struct {
|
||||
key string
|
||||
@@ -55,23 +51,10 @@ func (this *HTTPListener) Serve() error {
|
||||
switch state {
|
||||
case http.StateNew:
|
||||
atomic.AddInt64(&this.countActiveConnections, 1)
|
||||
|
||||
// 为指标存储连接信息
|
||||
if sharedNodeConfig.HasHTTPConnectionMetrics() {
|
||||
metricNewConnMapLocker.Lock()
|
||||
metricNewConnMap[conn.RemoteAddr().String()] = zero.New()
|
||||
metricNewConnMapLocker.Unlock()
|
||||
}
|
||||
case http.StateActive, http.StateIdle, http.StateHijacked:
|
||||
// Nothing to do
|
||||
case http.StateClosed:
|
||||
atomic.AddInt64(&this.countActiveConnections, -1)
|
||||
|
||||
// 移除指标存储连接信息
|
||||
// 因为中途配置可能有改变,所以暂时不添加条件
|
||||
metricNewConnMapLocker.Lock()
|
||||
delete(metricNewConnMap, conn.RemoteAddr().String())
|
||||
metricNewConnMapLocker.Unlock()
|
||||
}
|
||||
},
|
||||
ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/configs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
|
||||
@@ -21,6 +23,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/stats"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/clock"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf"
|
||||
"github.com/andybalholm/brotli"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
@@ -78,7 +81,7 @@ func (this *Node) Test() error {
|
||||
if err != nil {
|
||||
return errors.New("test rpc failed: " + err.Error())
|
||||
}
|
||||
_, err = rpcClient.APINodeRPC().FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
|
||||
_, err = rpcClient.APINodeRPC.FindCurrentAPINodeVersion(rpcClient.Context(), &pb.FindCurrentAPINodeVersionRequest{})
|
||||
if err != nil {
|
||||
return errors.New("test rpc failed: " + err.Error())
|
||||
}
|
||||
@@ -101,9 +104,6 @@ func (this *Node) Start() {
|
||||
// 监听signal
|
||||
this.listenSignals()
|
||||
|
||||
// 启动事件
|
||||
events.Notify(events.EventStart)
|
||||
|
||||
// 本地Sock
|
||||
err := this.listenSock()
|
||||
if err != nil {
|
||||
@@ -111,9 +111,19 @@ func (this *Node) Start() {
|
||||
return
|
||||
}
|
||||
|
||||
// 启动IP库
|
||||
remotelogs.Println("NODE", "initializing ip library ...")
|
||||
err = iplib.InitDefault()
|
||||
if err != nil {
|
||||
remotelogs.Error("NODE", "initialize ip library failed: "+err.Error())
|
||||
}
|
||||
|
||||
// 检查硬盘类型
|
||||
this.checkDisk()
|
||||
|
||||
// 启动事件
|
||||
events.Notify(events.EventStart)
|
||||
|
||||
// 读取API配置
|
||||
remotelogs.Println("NODE", "init config ...")
|
||||
err = this.syncConfig(0)
|
||||
@@ -146,11 +156,21 @@ func (this *Node) Start() {
|
||||
// 启动同步计时器
|
||||
this.startSyncTimer()
|
||||
|
||||
// 状态变更计时器
|
||||
// 更新IP库
|
||||
goman.New(func() {
|
||||
iplib.NewUpdater(NewIPLibraryUpdater(), 10*time.Minute).Start()
|
||||
})
|
||||
|
||||
// 监控节点运行状态
|
||||
goman.New(func() {
|
||||
NewNodeStatusExecutor().Listen()
|
||||
})
|
||||
|
||||
// 同步时间
|
||||
goman.New(func() {
|
||||
clock.Start()
|
||||
})
|
||||
|
||||
// 读取配置
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
@@ -294,7 +314,7 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
var nodeCtx = rpcClient.Context()
|
||||
tasksResp, err := rpcClient.NodeTaskRPC().FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
|
||||
tasksResp, err := rpcClient.NodeTaskRPC.FindNodeTasks(nodeCtx, &pb.FindNodeTasksRequest{})
|
||||
if err != nil {
|
||||
if rpc.IsConnError(err) && !Tea.IsTesting() {
|
||||
return nil
|
||||
@@ -312,7 +332,7 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -331,13 +351,13 @@ func (this *Node) loop() error {
|
||||
err = this.syncConfig(task.Version)
|
||||
}
|
||||
if err != nil {
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: false,
|
||||
Error: err.Error(),
|
||||
})
|
||||
} else {
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -359,7 +379,7 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -368,7 +388,7 @@ func (this *Node) loop() error {
|
||||
return err
|
||||
}
|
||||
case "nodeLevelChanged":
|
||||
levelInfoResp, err := rpcClient.NodeRPC().FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
|
||||
levelInfoResp, err := rpcClient.NodeRPC.FindNodeLevelInfo(nodeCtx, &pb.FindNodeLevelInfoRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -385,7 +405,7 @@ func (this *Node) loop() error {
|
||||
sharedNodeConfig.ParentNodes = parentNodes
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -394,7 +414,7 @@ func (this *Node) loop() error {
|
||||
return err
|
||||
}
|
||||
case "ddosProtectionChanged":
|
||||
resp, err := rpcClient.NodeRPC().FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
|
||||
resp, err := rpcClient.NodeRPC.FindNodeDDoSProtection(nodeCtx, &pb.FindNodeDDoSProtectionRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -421,7 +441,7 @@ func (this *Node) loop() error {
|
||||
}
|
||||
|
||||
// 修改为已同步
|
||||
_, err = rpcClient.NodeTaskRPC().ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
_, err = rpcClient.NodeTaskRPC.ReportNodeTaskDone(nodeCtx, &pb.ReportNodeTaskDoneRequest{
|
||||
NodeTaskId: task.Id,
|
||||
IsOk: true,
|
||||
Error: "",
|
||||
@@ -466,7 +486,7 @@ func (this *Node) syncConfig(taskVersion int64) error {
|
||||
nodeCtx := rpcClient.Context()
|
||||
|
||||
// TODO 这里考虑只同步版本号有变更的
|
||||
configResp, err := rpcClient.NodeRPC().FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
|
||||
configResp, err := rpcClient.NodeRPC.FindCurrentNodeConfig(nodeCtx, &pb.FindCurrentNodeConfigRequest{
|
||||
Version: -1, // 更新所有版本
|
||||
Compress: true,
|
||||
NodeTaskVersion: taskVersion,
|
||||
@@ -557,7 +577,7 @@ func (this *Node) syncServerConfig(serverId int64) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.ServerRPC().ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
|
||||
resp, err := rpcClient.ServerRPC.ComposeServerConfig(rpcClient.Context(), &pb.ComposeServerConfigRequest{ServerId: serverId})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -639,7 +659,7 @@ func (this *Node) checkClusterConfig() error {
|
||||
}
|
||||
|
||||
logs.Println("[NODE]registering node to cluster ...")
|
||||
resp, err := rpcClient.NodeRPC().RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
|
||||
resp, err := rpcClient.NodeRPC.RegisterClusterNode(rpcClient.ClusterContext(config.ClusterId, config.Secret), &pb.RegisterClusterNodeRequest{Name: HOSTNAME})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -778,13 +798,16 @@ func (this *Node) listenSock() error {
|
||||
},
|
||||
})
|
||||
case "conns":
|
||||
ipConns, serverConns := sharedClientConnLimiter.Conns()
|
||||
var addrs = []string{}
|
||||
var connMap = conns.SharedMap.AllConns()
|
||||
for _, conn := range connMap {
|
||||
addrs = append(addrs, conn.RemoteAddr().String())
|
||||
}
|
||||
|
||||
_ = cmd.Reply(&gosock.Command{
|
||||
Params: map[string]interface{}{
|
||||
"ipConns": ipConns,
|
||||
"serverConns": serverConns,
|
||||
"total": sharedListenerManager.TotalActiveConnections(),
|
||||
"addrs": addrs,
|
||||
"total": len(addrs),
|
||||
},
|
||||
})
|
||||
case "dropIP":
|
||||
@@ -856,6 +879,11 @@ func (this *Node) listenSock() error {
|
||||
} else {
|
||||
_ = cmd.ReplyOk()
|
||||
}
|
||||
case "bandwidth":
|
||||
var m = stats.SharedBandwidthStatManager.Map()
|
||||
_ = cmd.Reply(&gosock.Command{Params: maps.Map{
|
||||
"stats": m,
|
||||
}})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -866,7 +894,7 @@ func (this *Node) listenSock() error {
|
||||
})
|
||||
|
||||
events.OnKey(events.EventQuit, this, func() {
|
||||
logs.Println("NODE", "quit unix sock")
|
||||
remotelogs.Println("NODE", "quit unix sock")
|
||||
_ = this.sock.Close()
|
||||
})
|
||||
|
||||
@@ -1010,10 +1038,13 @@ func (this *Node) reloadServer() {
|
||||
}
|
||||
|
||||
func (this *Node) checkDisk() {
|
||||
if runtime.GOOS == "linux" {
|
||||
if runtime.GOOS != "linux" {
|
||||
return
|
||||
}
|
||||
for n := 'a'; n <= 'z'; n++ {
|
||||
for _, path := range []string{
|
||||
"/sys/block/vda/queue/rotational",
|
||||
"/sys/block/sda/queue/rotational",
|
||||
"/sys/block/vd" + string(n) + "/queue/rotational",
|
||||
"/sys/block/sd" + string(n) + "/queue/rotational",
|
||||
} {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
@@ -1022,7 +1053,7 @@ func (this *Node) checkDisk() {
|
||||
if string(data) == "0" {
|
||||
teaconst.DiskIsFast = true
|
||||
}
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,7 +125,7 @@ func (this *NodeStatusExecutor) update() {
|
||||
remotelogs.Error("NODE_STATUS", "failed to open rpc: "+err.Error())
|
||||
return
|
||||
}
|
||||
_, err = rpcClient.NodeRPC().UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
|
||||
_, err = rpcClient.NodeRPC.UpdateNodeStatus(rpcClient.Context(), &pb.UpdateNodeStatusRequest{
|
||||
StatusJSON: jsonData,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -60,7 +60,7 @@ func (this *OCSPUpdateTask) Loop() error {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := rpcClient.SSLCertRPC().ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
|
||||
resp, err := rpcClient.SSLCertRPC.ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
|
||||
Version: this.version,
|
||||
Size: 100,
|
||||
})
|
||||
|
||||
@@ -81,7 +81,7 @@ func (this *SyncAPINodesTask) Loop() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := rpcClient.APINodeRPC().FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
|
||||
resp, err := rpcClient.APINodeRPC.FindAllEnabledAPINodes(rpcClient.Context(), &pb.FindAllEnabledAPINodesRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -126,7 +126,7 @@ func (this *UpgradeManager) install() error {
|
||||
var sum = ""
|
||||
var filename = ""
|
||||
for {
|
||||
resp, err := client.NodeRPC().DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
|
||||
resp, err := client.NodeRPC.DownloadNodeInstallationFile(client.Context(), &pb.DownloadNodeInstallationFileRequest{
|
||||
Os: runtime.GOOS,
|
||||
Arch: runtime.GOARCH,
|
||||
ChunkOffset: offset,
|
||||
|
||||
@@ -173,7 +173,6 @@ func ServerSuccess(serverId int64, tag string, description string, logType nodec
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ServerLog 打印服务相关日志信息
|
||||
func ServerLog(serverId int64, tag string, description string, logType nodeconfigs.NodeLogType, params maps.Map) {
|
||||
logs.Println("[" + tag + "]" + description)
|
||||
@@ -253,6 +252,6 @@ Loop:
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = rpcClient.NodeLogRPC().CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
|
||||
_, err = rpcClient.NodeLogRPC.CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -28,6 +28,27 @@ type RPCClient struct {
|
||||
conns []*grpc.ClientConn
|
||||
|
||||
locker sync.RWMutex
|
||||
|
||||
NodeRPC pb.NodeServiceClient
|
||||
NodeLogRPC pb.NodeLogServiceClient
|
||||
NodeTaskRPC pb.NodeTaskServiceClient
|
||||
NodeValueRPC pb.NodeValueServiceClient
|
||||
HTTPAccessLogRPC pb.HTTPAccessLogServiceClient
|
||||
HTTPCacheTaskKeyRPC pb.HTTPCacheTaskKeyServiceClient
|
||||
APINodeRPC pb.APINodeServiceClient
|
||||
IPLibraryArtifactRPC pb.IPLibraryArtifactServiceClient
|
||||
IPListRPC pb.IPListServiceClient
|
||||
IPItemRPC pb.IPItemServiceClient
|
||||
FileRPC pb.FileServiceClient
|
||||
FileChunkRPC pb.FileChunkServiceClient
|
||||
ACMEAuthenticationRPC pb.ACMEAuthenticationServiceClient
|
||||
ServerRPC pb.ServerServiceClient
|
||||
ServerDailyStatRPC pb.ServerDailyStatServiceClient
|
||||
ServerBandwidthStatRPC pb.ServerBandwidthStatServiceClient
|
||||
MetricStatRPC pb.MetricStatServiceClient
|
||||
FirewallRPC pb.FirewallServiceClient
|
||||
SSLCertRPC pb.SSLCertServiceClient
|
||||
ScriptRPC pb.ScriptServiceClient
|
||||
}
|
||||
|
||||
func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
|
||||
@@ -35,10 +56,32 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
|
||||
return nil, errors.New("api config should not be nil")
|
||||
}
|
||||
|
||||
client := &RPCClient{
|
||||
var client = &RPCClient{
|
||||
apiConfig: apiConfig,
|
||||
}
|
||||
|
||||
// 初始化RPC实例
|
||||
client.NodeRPC = pb.NewNodeServiceClient(client)
|
||||
client.NodeLogRPC = pb.NewNodeLogServiceClient(client)
|
||||
client.NodeTaskRPC = pb.NewNodeTaskServiceClient(client)
|
||||
client.NodeValueRPC = pb.NewNodeValueServiceClient(client)
|
||||
client.HTTPAccessLogRPC = pb.NewHTTPAccessLogServiceClient(client)
|
||||
client.HTTPCacheTaskKeyRPC = pb.NewHTTPCacheTaskKeyServiceClient(client)
|
||||
client.APINodeRPC = pb.NewAPINodeServiceClient(client)
|
||||
client.IPLibraryArtifactRPC = pb.NewIPLibraryArtifactServiceClient(client)
|
||||
client.IPListRPC = pb.NewIPListServiceClient(client)
|
||||
client.IPItemRPC = pb.NewIPItemServiceClient(client)
|
||||
client.FileRPC = pb.NewFileServiceClient(client)
|
||||
client.FileChunkRPC = pb.NewFileChunkServiceClient(client)
|
||||
client.ACMEAuthenticationRPC = pb.NewACMEAuthenticationServiceClient(client)
|
||||
client.ServerRPC = pb.NewServerServiceClient(client)
|
||||
client.ServerDailyStatRPC = pb.NewServerDailyStatServiceClient(client)
|
||||
client.ServerBandwidthStatRPC = pb.NewServerBandwidthStatServiceClient(client)
|
||||
client.MetricStatRPC = pb.NewMetricStatServiceClient(client)
|
||||
client.FirewallRPC = pb.NewFirewallServiceClient(client)
|
||||
client.SSLCertRPC = pb.NewSSLCertServiceClient(client)
|
||||
client.ScriptRPC = pb.NewScriptServiceClient(client)
|
||||
|
||||
err := client.init()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -47,102 +90,10 @@ func NewRPCClient(apiConfig *configs.APIConfig) (*RPCClient, error) {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeRPC() pb.NodeServiceClient {
|
||||
return pb.NewNodeServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeLogRPC() pb.NodeLogServiceClient {
|
||||
return pb.NewNodeLogServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeTaskRPC() pb.NodeTaskServiceClient {
|
||||
return pb.NewNodeTaskServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) NodeValueRPC() pb.NodeValueServiceClient {
|
||||
return pb.NewNodeValueServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) HTTPAccessLogRPC() pb.HTTPAccessLogServiceClient {
|
||||
return pb.NewHTTPAccessLogServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) HTTPCacheTaskKeyRPC() pb.HTTPCacheTaskKeyServiceClient {
|
||||
return pb.NewHTTPCacheTaskKeyServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) APINodeRPC() pb.APINodeServiceClient {
|
||||
return pb.NewAPINodeServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPLibraryRPC() pb.IPLibraryServiceClient {
|
||||
return pb.NewIPLibraryServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionCountryRPC() pb.RegionCountryServiceClient {
|
||||
return pb.NewRegionCountryServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionProvinceRPC() pb.RegionProvinceServiceClient {
|
||||
return pb.NewRegionProvinceServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionCityRPC() pb.RegionCityServiceClient {
|
||||
return pb.NewRegionCityServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) RegionProviderRPC() pb.RegionProviderServiceClient {
|
||||
return pb.NewRegionProviderServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPListRPC() pb.IPListServiceClient {
|
||||
return pb.NewIPListServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) IPItemRPC() pb.IPItemServiceClient {
|
||||
return pb.NewIPItemServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) FileRPC() pb.FileServiceClient {
|
||||
return pb.NewFileServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) FileChunkRPC() pb.FileChunkServiceClient {
|
||||
return pb.NewFileChunkServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ACMEAuthenticationRPC() pb.ACMEAuthenticationServiceClient {
|
||||
return pb.NewACMEAuthenticationServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ServerRPC() pb.ServerServiceClient {
|
||||
return pb.NewServerServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ServerDailyStatRPC() pb.ServerDailyStatServiceClient {
|
||||
return pb.NewServerDailyStatServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) ServerBandwidthStatRPC() pb.ServerBandwidthStatServiceClient {
|
||||
return pb.NewServerBandwidthStatServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) MetricStatRPC() pb.MetricStatServiceClient {
|
||||
return pb.NewMetricStatServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) FirewallRPC() pb.FirewallServiceClient {
|
||||
return pb.NewFirewallServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
func (this *RPCClient) SSLCertRPC() pb.SSLCertServiceClient {
|
||||
return pb.NewSSLCertServiceClient(this.pickConn())
|
||||
}
|
||||
|
||||
// Context 节点上下文信息
|
||||
func (this *RPCClient) Context() context.Context {
|
||||
ctx := context.Background()
|
||||
m := maps.Map{
|
||||
var ctx = context.Background()
|
||||
var m = maps.Map{
|
||||
"timestamp": time.Now().Unix(),
|
||||
"type": "node",
|
||||
"userId": 0,
|
||||
@@ -157,7 +108,7 @@ func (this *RPCClient) Context() context.Context {
|
||||
utils.PrintError(err)
|
||||
return context.Background()
|
||||
}
|
||||
token := base64.StdEncoding.EncodeToString(data)
|
||||
var token = base64.StdEncoding.EncodeToString(data)
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, "nodeId", this.apiConfig.NodeId, "token", token)
|
||||
return ctx
|
||||
}
|
||||
@@ -209,7 +160,7 @@ func (this *RPCClient) UpdateConfig(config *configs.APIConfig) error {
|
||||
// 初始化
|
||||
func (this *RPCClient) init() error {
|
||||
// 重新连接
|
||||
conns := []*grpc.ClientConn{}
|
||||
var conns = []*grpc.ClientConn{}
|
||||
for _, endpoint := range this.apiConfig.RPC.Endpoints {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
@@ -248,20 +199,24 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
|
||||
|
||||
// 检查连接状态
|
||||
if len(this.conns) > 0 {
|
||||
availableConns := []*grpc.ClientConn{}
|
||||
for _, state := range []connectivity.State{connectivity.Ready, connectivity.Idle, connectivity.Connecting} {
|
||||
var availableConns = []*grpc.ClientConn{}
|
||||
for _, stateArray := range [][2]connectivity.State{
|
||||
{connectivity.Ready, connectivity.Idle}, // 优先Ready和Idle
|
||||
{connectivity.Connecting, connectivity.Connecting},
|
||||
} {
|
||||
for _, conn := range this.conns {
|
||||
if conn.GetState() == state {
|
||||
var state = conn.GetState()
|
||||
if state == stateArray[0] || state == stateArray[1] {
|
||||
availableConns = append(availableConns, conn)
|
||||
}
|
||||
}
|
||||
if len(availableConns) > 0 {
|
||||
break
|
||||
return this.randConn(availableConns)
|
||||
}
|
||||
}
|
||||
|
||||
if len(availableConns) > 0 {
|
||||
return availableConns[rands.Int(0, len(availableConns)-1)]
|
||||
return this.randConn(availableConns)
|
||||
}
|
||||
|
||||
// 关闭
|
||||
@@ -281,5 +236,31 @@ func (this *RPCClient) pickConn() *grpc.ClientConn {
|
||||
return nil
|
||||
}
|
||||
|
||||
return this.conns[rands.Int(0, len(this.conns)-1)]
|
||||
return this.randConn(this.conns)
|
||||
}
|
||||
|
||||
func (this *RPCClient) Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...grpc.CallOption) error {
|
||||
var conn = this.pickConn()
|
||||
if conn == nil {
|
||||
return errors.New("can not get available grpc connection")
|
||||
}
|
||||
return conn.Invoke(ctx, method, args, reply, opts...)
|
||||
}
|
||||
func (this *RPCClient) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) {
|
||||
var conn = this.pickConn()
|
||||
if conn == nil {
|
||||
return nil, errors.New("can not get available grpc connection")
|
||||
}
|
||||
return conn.NewStream(ctx, desc, method, opts...)
|
||||
}
|
||||
|
||||
func (this *RPCClient) randConn(conns []*grpc.ClientConn) *grpc.ClientConn {
|
||||
var l = len(conns)
|
||||
if l == 0 {
|
||||
return nil
|
||||
}
|
||||
if l == 1 {
|
||||
return conns[0]
|
||||
}
|
||||
return conns[rands.Int(0, l-1)]
|
||||
}
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
|
||||
var SharedBandwidthStatManager = NewBandwidthStatManager()
|
||||
|
||||
const bandwidthTimestampDelim = 2 // N秒平均,更为精确
|
||||
|
||||
func init() {
|
||||
events.On(events.EventLoaded, func() {
|
||||
goman.New(func() {
|
||||
@@ -83,7 +85,7 @@ func (this *BandwidthStatManager) Loop() error {
|
||||
ServerId: stat.ServerId,
|
||||
Day: stat.Day,
|
||||
TimeAt: stat.TimeAt,
|
||||
Bytes: stat.MaxBytes,
|
||||
Bytes: stat.MaxBytes / bandwidthTimestampDelim,
|
||||
})
|
||||
delete(this.m, key)
|
||||
}
|
||||
@@ -96,7 +98,7 @@ func (this *BandwidthStatManager) Loop() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = rpcClient.ServerBandwidthStatRPC().UploadServerBandwidthStats(rpcClient.Context(), &pb.UploadServerBandwidthStatsRequest{ServerBandwidthStats: pbStats})
|
||||
_, err = rpcClient.ServerBandwidthStatRPC.UploadServerBandwidthStats(rpcClient.Context(), &pb.UploadServerBandwidthStatsRequest{ServerBandwidthStats: pbStats})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -112,11 +114,18 @@ func (this *BandwidthStatManager) Add(userId int64, serverId int64, bytes int64)
|
||||
}
|
||||
|
||||
var now = time.Now()
|
||||
var timestamp = now.Unix()
|
||||
var timestamp = now.Unix() / bandwidthTimestampDelim * bandwidthTimestampDelim // 将时间戳均分成N等份
|
||||
var day = timeutil.Format("Ymd", now)
|
||||
var timeAt = timeutil.FormatTime("Hi", now.Unix()/300*300)
|
||||
var key = types.String(serverId) + "@" + day + "@" + timeAt
|
||||
|
||||
// 增加TCP Header尺寸,这里默认MTU为1500,且默认为IPv4
|
||||
const mtu = 1500
|
||||
const tcpHeaderSize = 20
|
||||
if bytes > mtu {
|
||||
bytes += bytes * tcpHeaderSize / mtu
|
||||
}
|
||||
|
||||
this.locker.Lock()
|
||||
stat, ok := this.m[key]
|
||||
if ok {
|
||||
@@ -150,3 +159,15 @@ func (this *BandwidthStatManager) Inspect() {
|
||||
logs.PrintAsJSON(this.m)
|
||||
this.locker.Unlock()
|
||||
}
|
||||
|
||||
func (this *BandwidthStatManager) Map() map[int64]int64 /** serverId => max bytes **/ {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
|
||||
var m = map[int64]int64{}
|
||||
for _, v := range this.m {
|
||||
m[v.ServerId] = v.MaxBytes / bandwidthTimestampDelim
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestBandwidthStatManager_Add(t *testing.T) {
|
||||
manager.Add(1, 1, 10)
|
||||
manager.Add(1, 1, 10)
|
||||
time.Sleep(1 * time.Second)
|
||||
manager.Add(1, 1, 15)
|
||||
manager.Add(1, 1, 85)
|
||||
time.Sleep(1 * time.Second)
|
||||
manager.Add(1, 1, 25)
|
||||
manager.Add(1, 1, 75)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/monitor"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
@@ -190,31 +190,26 @@ Loop:
|
||||
var serverId = pieces[0]
|
||||
var ip = pieces[1]
|
||||
|
||||
if iplibrary.SharedLibrary != nil {
|
||||
result, err := iplibrary.SharedLibrary.Lookup(ip)
|
||||
if err == nil && result != nil {
|
||||
if len(result.Country) == 0 {
|
||||
continue
|
||||
}
|
||||
var result = iplib.LookupIP(ip)
|
||||
if result != nil && result.IsOk() {
|
||||
var key = serverId + "@" + result.CountryName() + "@" + result.ProvinceName() + "@" + result.CityName()
|
||||
stat, ok := this.cityMap[key]
|
||||
if !ok {
|
||||
stat = &StatItem{}
|
||||
this.cityMap[key] = stat
|
||||
}
|
||||
stat.Bytes += types.Int64(pieces[2])
|
||||
stat.CountRequests++
|
||||
if types.Int8(pieces[3]) == 1 {
|
||||
stat.AttackBytes += types.Int64(pieces[2])
|
||||
stat.CountAttackRequests++
|
||||
}
|
||||
|
||||
var key = serverId + "@" + result.Country + "@" + result.Province + "@" + result.City
|
||||
stat, ok := this.cityMap[key]
|
||||
if !ok {
|
||||
stat = &StatItem{}
|
||||
this.cityMap[key] = stat
|
||||
}
|
||||
stat.Bytes += types.Int64(pieces[2])
|
||||
stat.CountRequests++
|
||||
if types.Int8(pieces[3]) == 1 {
|
||||
stat.AttackBytes += types.Int64(pieces[2])
|
||||
stat.CountAttackRequests++
|
||||
}
|
||||
|
||||
if len(result.ISP) > 0 {
|
||||
this.providerMap[serverId+"@"+result.ISP]++
|
||||
}
|
||||
if len(result.ProviderName()) > 0 {
|
||||
this.providerMap[serverId+"@"+result.ProviderName()]++
|
||||
}
|
||||
}
|
||||
|
||||
case userAgentString := <-this.userAgentChan:
|
||||
var atIndex = strings.Index(userAgentString, "@")
|
||||
if atIndex < 0 {
|
||||
@@ -329,7 +324,7 @@ func (this *HTTPRequestStatManager) Upload() error {
|
||||
this.dailyFirewallRuleGroupMap = map[string]int64{}
|
||||
|
||||
// 上传数据
|
||||
_, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
|
||||
_, err = rpcClient.ServerRPC.UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
|
||||
Month: timeutil.Format("Ym"),
|
||||
Day: timeutil.Format("Ymd"),
|
||||
RegionCities: pbCities,
|
||||
@@ -351,7 +346,7 @@ func (this *HTTPRequestStatManager) Upload() error {
|
||||
}
|
||||
|
||||
// 再次尝试
|
||||
_, err = rpcClient.ServerRPC().UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
|
||||
_, err = rpcClient.ServerRPC.UploadServerHTTPRequestStat(rpcClient.Context(), &pb.UploadServerHTTPRequestStatRequest{
|
||||
Month: timeutil.Format("Ym"),
|
||||
Day: timeutil.Format("Ymd"),
|
||||
RegionCities: pbCities,
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
|
||||
iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHTTPRequestStatManager_Loop_Region(t *testing.T) {
|
||||
library, err := iplibrary.SharedManager.Load()
|
||||
err := iplib.InitDefault()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
iplibrary.SharedLibrary = library
|
||||
|
||||
manager := NewHTTPRequestStatManager()
|
||||
var manager = NewHTTPRequestStatManager()
|
||||
manager.AddRemoteAddr(11, "202.196.0.20", 0, false)
|
||||
manager.AddRemoteAddr(11, "202.196.0.20", 0, false) // 重复添加一个测试相加
|
||||
manager.AddRemoteAddr(11, "8.8.8.8", 0, false)
|
||||
@@ -36,19 +36,13 @@ func TestHTTPRequestStatManager_Loop_Region(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHTTPRequestStatManager_Loop_UserAgent(t *testing.T) {
|
||||
library, err := iplibrary.SharedManager.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
iplibrary.SharedLibrary = library
|
||||
|
||||
manager := NewHTTPRequestStatManager()
|
||||
var manager = NewHTTPRequestStatManager()
|
||||
manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
|
||||
manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
|
||||
manager.AddUserAgent(1, "Mozilla/5.0 (Macintosh; Intel Mac OS X 11) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76 Safari/537.36")
|
||||
manager.AddUserAgent(1, "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36")
|
||||
manager.AddUserAgent(1, "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
|
||||
err = manager.Loop()
|
||||
err := manager.Loop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -210,7 +210,7 @@ func (this *TrafficStatManager) Upload() error {
|
||||
})
|
||||
}
|
||||
|
||||
_, err = client.ServerDailyStatRPC().UploadServerDailyStats(client.Context(), &pb.UploadServerDailyStatsRequest{
|
||||
_, err = client.ServerDailyStatRPC.UploadServerDailyStats(client.Context(), &pb.UploadServerDailyStatsRequest{
|
||||
Stats: pbServerStats,
|
||||
DomainStats: pbDomainStats,
|
||||
})
|
||||
|
||||
58
internal/utils/clock/utils.go
Normal file
58
internal/utils/clock/utils.go
Normal file
@@ -0,0 +1,58 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package clock
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Start TODO 需要可以在集群中配置
|
||||
func Start() {
|
||||
// sync once
|
||||
err := Sync()
|
||||
if err != nil {
|
||||
remotelogs.Warn("CLOCK", "sync time clock failed: "+err.Error())
|
||||
}
|
||||
|
||||
var ticker = time.NewTicker(1 * time.Hour)
|
||||
for range ticker.C {
|
||||
err := Sync()
|
||||
if err != nil {
|
||||
// ignore error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sync 自动校对时间
|
||||
func Sync() error {
|
||||
if runtime.GOOS != "linux" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ntpdate, err := exec.LookPath("ntpdate")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if len(ntpdate) > 0 {
|
||||
return syncNtpdate(ntpdate)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncNtpdate(ntpdate string) error {
|
||||
var cmd = exec.Command(ntpdate, "pool.ntp.org")
|
||||
var stderr = &bytes.Buffer{}
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.New(err.Error() + ": " + stderr.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
155
internal/utils/dbs/batch.go
Normal file
155
internal/utils/dbs/batch.go
Normal file
@@ -0,0 +1,155 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package dbs
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"time"
|
||||
)
|
||||
|
||||
type batchItem struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type Batch struct {
|
||||
db *sql.DB
|
||||
n int
|
||||
|
||||
enableStat bool
|
||||
|
||||
onFail func(err error)
|
||||
|
||||
queue chan *batchItem
|
||||
close chan bool
|
||||
|
||||
isClosed bool
|
||||
}
|
||||
|
||||
func NewBatch(db *sql.DB, n int) *Batch {
|
||||
return &Batch{
|
||||
db: db,
|
||||
n: n,
|
||||
queue: make(chan *batchItem),
|
||||
close: make(chan bool, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Batch) EnableStat(b bool) {
|
||||
this.enableStat = b
|
||||
}
|
||||
|
||||
func (this *Batch) OnFail(callback func(err error)) {
|
||||
this.onFail = callback
|
||||
}
|
||||
|
||||
func (this *Batch) Add(query string, args ...any) {
|
||||
this.queue <- &batchItem{
|
||||
query: query,
|
||||
args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Batch) Exec() {
|
||||
var n = this.n
|
||||
if n <= 0 {
|
||||
n = 4
|
||||
}
|
||||
|
||||
var ticker = time.NewTicker(100 * time.Millisecond)
|
||||
var count = 0
|
||||
var lastTx *sql.Tx
|
||||
For:
|
||||
for {
|
||||
// closed
|
||||
if this.isClosed {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case item := <-this.queue:
|
||||
if lastTx == nil {
|
||||
lastTx = this.beginTx()
|
||||
if lastTx == nil {
|
||||
continue For
|
||||
}
|
||||
}
|
||||
|
||||
err := this.execItem(lastTx, item)
|
||||
if err != nil {
|
||||
this.processErr(item.query, err)
|
||||
}
|
||||
|
||||
count++
|
||||
|
||||
if count == n {
|
||||
count = 0
|
||||
err = lastTx.Commit()
|
||||
lastTx = nil
|
||||
if err != nil {
|
||||
this.processErr("commit", err)
|
||||
}
|
||||
}
|
||||
case <-ticker.C:
|
||||
if lastTx == nil || count == 0 {
|
||||
continue For
|
||||
}
|
||||
count = 0
|
||||
err := lastTx.Commit()
|
||||
lastTx = nil
|
||||
if err != nil {
|
||||
this.processErr("commit", err)
|
||||
}
|
||||
case <-this.close:
|
||||
// closed
|
||||
|
||||
if lastTx != nil {
|
||||
_ = lastTx.Commit()
|
||||
lastTx = nil
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Batch) Close() {
|
||||
this.isClosed = true
|
||||
|
||||
select {
|
||||
case this.close <- true:
|
||||
default:
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Batch) beginTx() *sql.Tx {
|
||||
tx, err := this.db.Begin()
|
||||
if err != nil {
|
||||
this.processErr("begin transaction", err)
|
||||
return nil
|
||||
}
|
||||
return tx
|
||||
}
|
||||
|
||||
func (this *Batch) execItem(tx *sql.Tx, item *batchItem) error {
|
||||
if this.enableStat {
|
||||
defer SharedQueryStatManager.AddQuery(item.query).End()
|
||||
}
|
||||
|
||||
_, err := tx.Exec(item.query, item.args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (this *Batch) processErr(prefix string, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if this.onFail != nil {
|
||||
this.onFail(err)
|
||||
} else {
|
||||
remotelogs.Error("SQLITE_BATCH", prefix+": "+err.Error())
|
||||
}
|
||||
}
|
||||
@@ -80,3 +80,7 @@ func (this *DB) Close() error {
|
||||
events.Remove(fmt.Sprintf("db_%p", this))
|
||||
return this.rawDB.Close()
|
||||
}
|
||||
|
||||
func (this *DB) RawDB() *sql.DB {
|
||||
return this.rawDB
|
||||
}
|
||||
|
||||
@@ -77,6 +77,12 @@ func (this *List) Remove(itemId uint64) {
|
||||
this.removeItem(itemId)
|
||||
}
|
||||
|
||||
func (this *List) ExpiresAt(itemId uint64) int64 {
|
||||
this.locker.Lock()
|
||||
defer this.locker.Unlock()
|
||||
return this.itemsMap[itemId]
|
||||
}
|
||||
|
||||
func (this *List) GC(timestamp int64) ItemMap {
|
||||
if this.lastTimestamp > timestamp+1 {
|
||||
return nil
|
||||
|
||||
@@ -25,7 +25,7 @@ func (this *AllowAction) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *AllowAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *AllowAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// do nothing
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
@@ -58,11 +58,11 @@ func (this *BlockAction) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// 加入到黑名单
|
||||
var timeout = this.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 60 // 默认封锁60秒
|
||||
timeout = 300 // 默认封锁300秒
|
||||
}
|
||||
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+int64(timeout), waf.Id, waf.UseLocalFirewall, group.Id, set.Id, "")
|
||||
@@ -82,14 +82,14 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
|
||||
req, err := http.NewRequest(http.MethodGet, this.URL, nil)
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
req.Header.Set("User-Agent", teaconst.GlobalProductName+"/"+teaconst.Version)
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
@@ -113,11 +113,11 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
_, _ = writer.Write(data)
|
||||
}
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
if len(this.Body) > 0 {
|
||||
_, _ = writer.Write([]byte(this.Body))
|
||||
@@ -126,5 +126,5 @@ func (this *BlockAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, reque
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -98,10 +98,10 @@ func (this *CaptchaAction) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// 是否在白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, req.WAFServerId(), req.WAFRemoteIP()) {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
var refURL = req.WAFRaw().URL.String()
|
||||
@@ -128,7 +128,7 @@ func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
info, err := utils.SimpleEncryptMap(captchaConfig)
|
||||
if err != nil {
|
||||
remotelogs.Error("WAF_CAPTCHA_ACTION", "encode captcha config failed: "+err.Error())
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
// 占用一次失败次数
|
||||
@@ -136,5 +136,5 @@ func (this *CaptchaAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
|
||||
http.Redirect(writer, req.WAFRaw(), CaptchaPath+"?info="+url.QueryEscape(info), http.StatusTemporaryRedirect)
|
||||
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -41,15 +41,15 @@ func (this *Get302Action) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// 仅限于Get
|
||||
if request.WAFRaw().Method != http.MethodGet {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
// 是否已经在白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
var m = maps.Map{
|
||||
@@ -64,7 +64,7 @@ func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
|
||||
info, err := utils.SimpleEncryptMap(m)
|
||||
if err != nil {
|
||||
remotelogs.Error("WAF_GET_302_ACTION", "encode info failed: "+err.Error())
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
http.Redirect(writer, request.WAFRaw(), Get302Path+"?info="+url.QueryEscape(info), http.StatusFound)
|
||||
@@ -73,5 +73,5 @@ func (this *Get302Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
|
||||
_ = this.CloseConn(writer)
|
||||
}
|
||||
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package waf
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
)
|
||||
@@ -29,20 +29,20 @@ func (this *GoGroupAction) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *GoGroupAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *GoGroupAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
nextGroup := waf.FindRuleGroup(types.Int64(this.GroupId))
|
||||
if nextGroup == nil || !nextGroup.IsOn {
|
||||
return true
|
||||
return true, true
|
||||
}
|
||||
|
||||
b, _, nextSet, err := nextGroup.MatchRequest(request)
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
return true
|
||||
remotelogs.Error("WAF", "GO_GROUP_ACTION: "+err.Error())
|
||||
return true, false
|
||||
}
|
||||
|
||||
if !b {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
return nextSet.PerformActions(waf, nextGroup, request, writer)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package waf
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
)
|
||||
@@ -30,23 +30,23 @@ func (this *GoSetAction) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *GoSetAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *GoSetAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
nextGroup := waf.FindRuleGroup(types.Int64(this.GroupId))
|
||||
if nextGroup == nil || !nextGroup.IsOn {
|
||||
return true
|
||||
return true, true
|
||||
}
|
||||
nextSet := nextGroup.FindRuleSet(types.Int64(this.SetId))
|
||||
if nextSet == nil || !nextSet.IsOn {
|
||||
return true
|
||||
return true, true
|
||||
}
|
||||
|
||||
b, _, err := nextSet.MatchRequest(request)
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
return true
|
||||
remotelogs.Error("WAF", "GO_GROUP_ACTION: "+err.Error())
|
||||
return true, false
|
||||
}
|
||||
if !b {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
return nextSet.PerformActions(waf, nextGroup, request, writer)
|
||||
}
|
||||
|
||||
@@ -27,5 +27,5 @@ type ActionInterface interface {
|
||||
WillChange() bool
|
||||
|
||||
// Perform the action
|
||||
Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool)
|
||||
Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool)
|
||||
}
|
||||
|
||||
133
internal/waf/action_js_cookie.go
Normal file
133
internal/waf/action_js_cookie.go
Normal file
@@ -0,0 +1,133 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package waf
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/waf/requests"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type JSCookieAction struct {
|
||||
BaseAction
|
||||
|
||||
Life int32 `yaml:"life" json:"life"`
|
||||
MaxFails int `yaml:"maxFails" json:"maxFails"` // 最大失败次数
|
||||
FailBlockTimeout int `yaml:"failBlockTimeout" json:"failBlockTimeout"` // 失败拦截时间
|
||||
Scope string `yaml:"scope" json:"scope"`
|
||||
}
|
||||
|
||||
func (this *JSCookieAction) Init(waf *WAF) error {
|
||||
this.Scope = firewallconfigs.FirewallScopeGlobal
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *JSCookieAction) Code() string {
|
||||
return ActionJavascriptCookie
|
||||
}
|
||||
|
||||
func (this *JSCookieAction) IsAttack() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *JSCookieAction) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *JSCookieAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// 是否在白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, req.WAFServerId(), req.WAFRemoteIP()) {
|
||||
return true, false
|
||||
}
|
||||
|
||||
nodeConfig, err := nodeconfigs.SharedNodeConfig()
|
||||
if err != nil {
|
||||
return true, false
|
||||
}
|
||||
|
||||
var life = this.Life
|
||||
if life <= 0 {
|
||||
life = 3600
|
||||
}
|
||||
|
||||
// 检查Cookie
|
||||
var cookieName = "ge_js_validator_" + types.String(set.Id)
|
||||
cookie, err := req.WAFRaw().Cookie(cookieName)
|
||||
if err == nil && cookie != nil {
|
||||
var cookieValue = cookie.Value
|
||||
if len(cookieValue) > 10 {
|
||||
var pieces = strings.Split(cookieValue, "@")
|
||||
if len(pieces) == 3 {
|
||||
var timestamp = pieces[0]
|
||||
var sum = pieces[2]
|
||||
if types.Int64(timestamp) >= time.Now().Unix()-int64(life) && fmt.Sprintf("%x", md5.Sum([]byte(timestamp+"@"+types.String(set.Id)+"@"+nodeConfig.NodeId))) == sum {
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
writer.Header().Set("Cache-Control", "no-cache")
|
||||
|
||||
var timestamp = types.String(time.Now().Unix())
|
||||
|
||||
var cookieValue = timestamp + "@" + types.String(set.Id) + "@" + fmt.Sprintf("%x", md5.Sum([]byte(timestamp+"@"+types.String(set.Id)+"@"+nodeConfig.NodeId)))
|
||||
|
||||
_, _ = writer.Write([]byte(`<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title></title>
|
||||
<meta charset="UTF-8"/>
|
||||
<script type="text/javascript">
|
||||
document.cookie = "` + cookieName + `=` + cookieValue + `; path=/; max-age=` + types.String(life) + `;";
|
||||
window.location.reload();
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
</body>
|
||||
</html>`))
|
||||
|
||||
// 记录失败次数
|
||||
this.increaseFails(req, waf.Id, group.Id, set.Id)
|
||||
|
||||
return false, false
|
||||
}
|
||||
|
||||
func (this *JSCookieAction) increaseFails(req requests.Request, policyId int64, groupId int64, setId int64) (goNext bool) {
|
||||
var maxFails = this.MaxFails
|
||||
var failBlockTimeout = this.FailBlockTimeout
|
||||
|
||||
if maxFails <= 0 {
|
||||
maxFails = 10 // 默认10次
|
||||
} else if maxFails <= 3 {
|
||||
maxFails = 3 // 不能小于3,防止意外刷新出现
|
||||
}
|
||||
if failBlockTimeout <= 0 {
|
||||
failBlockTimeout = 1800 // 默认1800s
|
||||
}
|
||||
|
||||
var key = "JS_COOKIE:FAILS:" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + req.WAFRaw().URL.String()
|
||||
|
||||
var countFails = ttlcache.SharedCache.IncreaseInt64(key, 1, time.Now().Unix()+300, true)
|
||||
if int(countFails) >= maxFails {
|
||||
var useLocalFirewall = false
|
||||
|
||||
if this.Scope == firewallconfigs.FirewallScopeGlobal {
|
||||
useLocalFirewall = true
|
||||
}
|
||||
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, useLocalFirewall, groupId, setId, "JS_COOKIE验证连续失败超过"+types.String(maxFails)+"次")
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
@@ -25,6 +25,6 @@ func (this *LogAction) WillChange() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *LogAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
return true
|
||||
func (this *LogAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
return true, false
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ func init() {
|
||||
}
|
||||
|
||||
for task := range notifyChan {
|
||||
_, err = rpcClient.FirewallRPC().NotifyHTTPFirewallEvent(rpcClient.Context(), &pb.NotifyHTTPFirewallEventRequest{
|
||||
_, err = rpcClient.FirewallRPC.NotifyHTTPFirewallEvent(rpcClient.Context(), &pb.NotifyHTTPFirewallEventRequest{
|
||||
ServerId: task.ServerId,
|
||||
HttpFirewallPolicyId: task.HttpFirewallPolicyId,
|
||||
HttpFirewallRuleGroupId: task.HttpFirewallRuleGroupId,
|
||||
@@ -71,7 +71,7 @@ func (this *NotifyAction) WillChange() bool {
|
||||
}
|
||||
|
||||
// Perform the action
|
||||
func (this *NotifyAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *NotifyAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
select {
|
||||
case notifyChan <- ¬ifyTask{
|
||||
ServerId: request.WAFServerId(),
|
||||
@@ -84,5 +84,5 @@ func (this *NotifyAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, requ
|
||||
|
||||
}
|
||||
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
@@ -32,10 +32,10 @@ func (this *PageAction) WillChange() bool {
|
||||
}
|
||||
|
||||
// Perform the action
|
||||
func (this *PageAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *PageAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
writer.WriteHeader(this.Status)
|
||||
_, _ = writer.Write([]byte(request.Format(this.Body)))
|
||||
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -33,17 +33,17 @@ func (this *Post307Action) WillChange() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
var cookieName = "WAF_VALIDATOR_ID"
|
||||
|
||||
// 仅限于POST
|
||||
if request.WAFRaw().Method != http.MethodPost {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
// 是否已经在白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
// 判断是否有Cookie
|
||||
@@ -57,7 +57,7 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
}
|
||||
var setId = m.GetString("setId")
|
||||
SharedIPWhiteList.RecordIP("set:"+setId, this.Scope, request.WAFServerId(), request.WAFRemoteIP(), time.Now().Unix()+life, m.GetInt64("policyId"), false, m.GetInt64("groupId"), m.GetInt64("setId"), "")
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
info, err := utils.SimpleEncryptMap(m)
|
||||
if err != nil {
|
||||
remotelogs.Error("WAF_POST_302_ACTION", "encode info failed: "+err.Error())
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
// 设置Cookie
|
||||
@@ -90,5 +90,5 @@ func (this *Post307Action) Perform(waf *WAF, group *RuleGroup, set *RuleSet, req
|
||||
_ = this.CloseConn(writer)
|
||||
}
|
||||
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ func init() {
|
||||
if len(reason) == 0 {
|
||||
reason = "触发WAF规则自动加入"
|
||||
}
|
||||
_, err = rpcClient.IPItemRPC().CreateIPItem(rpcClient.Context(), &pb.CreateIPItemRequest{
|
||||
_, err = rpcClient.IPItemRPC.CreateIPItem(rpcClient.Context(), &pb.CreateIPItemRequest{
|
||||
IpListId: task.listId,
|
||||
IpFrom: task.ip,
|
||||
IpTo: "",
|
||||
@@ -99,10 +99,10 @@ func (this *RecordIPAction) WillChange() bool {
|
||||
return this.Type == "black"
|
||||
}
|
||||
|
||||
func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
// 是否在本地白名单中
|
||||
if SharedIPWhiteList.Contains("set:"+types.String(set.Id), this.Scope, request.WAFServerId(), request.WAFRemoteIP()) {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
timeout := this.Timeout
|
||||
@@ -147,5 +147,5 @@ func (this *RecordIPAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, re
|
||||
}
|
||||
}
|
||||
|
||||
return this.Type != "black"
|
||||
return this.Type != "black", false
|
||||
}
|
||||
|
||||
@@ -27,6 +27,6 @@ func (this *TagAction) WillChange() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (this *TagAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (allow bool) {
|
||||
return true
|
||||
func (this *TagAction) Perform(waf *WAF, group *RuleGroup, set *RuleSet, request requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
return true, true
|
||||
}
|
||||
|
||||
@@ -5,18 +5,19 @@ import "reflect"
|
||||
type ActionString = string
|
||||
|
||||
const (
|
||||
ActionLog ActionString = "log" // allow and log
|
||||
ActionBlock ActionString = "block" // block
|
||||
ActionCaptcha ActionString = "captcha" // block and show captcha
|
||||
ActionNotify ActionString = "notify" // 告警
|
||||
ActionGet302 ActionString = "get_302" // 针对GET的302重定向认证
|
||||
ActionPost307 ActionString = "post_307" // 针对POST的307重定向认证
|
||||
ActionRecordIP ActionString = "record_ip" // 记录IP
|
||||
ActionTag ActionString = "tag" // 标签
|
||||
ActionPage ActionString = "page" // 显示网页
|
||||
ActionAllow ActionString = "allow" // allow
|
||||
ActionGoGroup ActionString = "go_group" // go to next rule group
|
||||
ActionGoSet ActionString = "go_set" // go to next rule set
|
||||
ActionLog ActionString = "log" // allow and log
|
||||
ActionBlock ActionString = "block" // block
|
||||
ActionCaptcha ActionString = "captcha" // block and show captcha
|
||||
ActionJavascriptCookie ActionString = "js_cookie" // js cookie
|
||||
ActionNotify ActionString = "notify" // 告警
|
||||
ActionGet302 ActionString = "get_302" // 针对GET的302重定向认证
|
||||
ActionPost307 ActionString = "post_307" // 针对POST的307重定向认证
|
||||
ActionRecordIP ActionString = "record_ip" // 记录IP
|
||||
ActionTag ActionString = "tag" // 标签
|
||||
ActionPage ActionString = "page" // 显示网页
|
||||
ActionAllow ActionString = "allow" // allow
|
||||
ActionGoGroup ActionString = "go_group" // go to next rule group
|
||||
ActionGoSet ActionString = "go_set" // go to next rule set
|
||||
)
|
||||
|
||||
var AllActions = []*ActionDefinition{
|
||||
@@ -44,6 +45,12 @@ var AllActions = []*ActionDefinition{
|
||||
Instance: new(CaptchaAction),
|
||||
Type: reflect.TypeOf(new(CaptchaAction)).Elem(),
|
||||
},
|
||||
{
|
||||
Name: "JS Cookie验证",
|
||||
Code: ActionJavascriptCookie,
|
||||
Instance: new(JSCookieAction),
|
||||
Type: reflect.TypeOf(new(JSCookieAction)).Elem(),
|
||||
},
|
||||
{
|
||||
Name: "告警",
|
||||
Code: ActionNotify,
|
||||
|
||||
@@ -34,7 +34,7 @@ func CaptchaIncreaseFails(req requests.Request, actionConfig *CaptchaAction, pol
|
||||
useLocalFirewall = true
|
||||
}
|
||||
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, useLocalFirewall, groupId, setId, "CAPTCHA验证连续失败")
|
||||
SharedIPBlackList.RecordIP(IPTypeAll, firewallconfigs.FirewallScopeService, req.WAFServerId(), req.WAFRemoteIP(), time.Now().Unix()+int64(failBlockTimeout), policyId, useLocalFirewall, groupId, setId, "CAPTCHA验证连续失败超过"+types.String(maxFails)+"次")
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -50,5 +50,5 @@ func CaptchaDeleteCacheKey(req requests.Request) {
|
||||
|
||||
// CaptchaCacheKey 获取Captcha缓存Key
|
||||
func CaptchaCacheKey(req requests.Request, pageCode CaptchaPageCode) string {
|
||||
return "CAPTCHA:FAILS:" + pageCode + ":" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId())
|
||||
return "CAPTCHA:FAILS:" + pageCode + ":" + req.WAFRemoteIP() + ":" + types.String(req.WAFServerId()) + ":" + req.WAFRaw().URL.String()
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ package waf
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/expires"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
@@ -47,7 +48,7 @@ func NewIPList(listType IPListType) *IPList {
|
||||
list.expireList = e
|
||||
|
||||
e.OnGC(func(itemId uint64) {
|
||||
list.remove(itemId)
|
||||
list.remove(itemId) // TODO 使用异步,防止阻塞GC
|
||||
})
|
||||
|
||||
return list
|
||||
@@ -67,6 +68,14 @@ func (this *IPList) Add(ipType string, scope firewallconfigs.FirewallScope, serv
|
||||
var id = this.nextId()
|
||||
this.expireList.Add(id, expiresAt)
|
||||
this.locker.Lock()
|
||||
|
||||
// 删除以前
|
||||
oldId, ok := this.ipMap[ip]
|
||||
if ok {
|
||||
delete(this.idMap, oldId)
|
||||
this.expireList.Remove(oldId)
|
||||
}
|
||||
|
||||
this.ipMap[ip] = id
|
||||
this.idMap[id] = ip
|
||||
this.locker.Unlock()
|
||||
@@ -115,6 +124,9 @@ func (this *IPList) RecordIP(ipType string,
|
||||
_ = firewalls.Firewall().DropSourceIP(ip, int(seconds), true)
|
||||
}
|
||||
}
|
||||
|
||||
// 关闭此IP相关连接
|
||||
conns.SharedMap.CloseIPConns(ip)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,13 +147,52 @@ func (this *IPList) Contains(ipType string, scope firewallconfigs.FirewallScope,
|
||||
return ok
|
||||
}
|
||||
|
||||
// ContainsExpires 判断是否有某个IP,并返回过期时间
|
||||
func (this *IPList) ContainsExpires(ipType string, scope firewallconfigs.FirewallScope, serverId int64, ip string) (expiresAt int64, ok bool) {
|
||||
switch scope {
|
||||
case firewallconfigs.FirewallScopeGlobal:
|
||||
ip = "*@" + ip + "@" + ipType
|
||||
case firewallconfigs.FirewallScopeService:
|
||||
ip = types.String(serverId) + "@" + ip + "@" + ipType
|
||||
default:
|
||||
ip = "*@" + ip + "@" + ipType
|
||||
}
|
||||
|
||||
this.locker.RLock()
|
||||
id, ok := this.ipMap[ip]
|
||||
if ok {
|
||||
expiresAt = this.expireList.ExpiresAt(id)
|
||||
}
|
||||
this.locker.RUnlock()
|
||||
return expiresAt, ok
|
||||
}
|
||||
|
||||
// RemoveIP 删除IP
|
||||
func (this *IPList) RemoveIP(ip string, serverId int64, shouldExecute bool) {
|
||||
this.locker.Lock()
|
||||
delete(this.ipMap, "*@"+ip+"@"+IPTypeAll)
|
||||
if serverId > 0 {
|
||||
delete(this.ipMap, types.String(serverId)+"@"+ip+"@"+IPTypeAll)
|
||||
|
||||
{
|
||||
var key = "*@" + ip + "@" + IPTypeAll
|
||||
id, ok := this.ipMap[key]
|
||||
if ok {
|
||||
delete(this.ipMap, key)
|
||||
delete(this.idMap, id)
|
||||
|
||||
this.expireList.Remove(id)
|
||||
}
|
||||
}
|
||||
|
||||
if serverId > 0 {
|
||||
var key = types.String(serverId) + "@" + ip + "@" + IPTypeAll
|
||||
id, ok := this.ipMap[key]
|
||||
if ok {
|
||||
delete(this.ipMap, key)
|
||||
delete(this.idMap, id)
|
||||
|
||||
this.expireList.Remove(id)
|
||||
}
|
||||
}
|
||||
|
||||
this.locker.Unlock()
|
||||
|
||||
// 从本地防火墙中删除
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"testing"
|
||||
@@ -13,18 +14,34 @@ import (
|
||||
)
|
||||
|
||||
func TestNewIPList(t *testing.T) {
|
||||
list := NewIPList(IPListTypeDeny)
|
||||
var list = NewIPList(IPListTypeDeny)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix())
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.2", time.Now().Unix()+1)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix()+2)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeService, 1, "127.0.0.3", time.Now().Unix()+3)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.10", time.Now().Unix()+10)
|
||||
|
||||
list.RemoveIP("127.0.0.1", 1, false)
|
||||
|
||||
logs.PrintAsJSON(list.ipMap, t)
|
||||
logs.PrintAsJSON(list.idMap, t)
|
||||
}
|
||||
|
||||
func TestIPList_Expire(t *testing.T) {
|
||||
var list = NewIPList(IPListTypeDeny)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix())
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.2", time.Now().Unix()+1)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.1", time.Now().Unix()+2)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.3", time.Now().Unix()+3)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.10", time.Now().Unix()+10)
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "127.0.0.10", time.Now().Unix()+6)
|
||||
|
||||
var ticker = time.NewTicker(1 * time.Second)
|
||||
for range ticker.C {
|
||||
t.Log("====")
|
||||
list.locker.Lock()
|
||||
logs.PrintAsJSON(list.ipMap, t)
|
||||
logs.PrintAsJSON(list.idMap, t)
|
||||
list.locker.Unlock()
|
||||
if len(list.idMap) == 0 {
|
||||
break
|
||||
}
|
||||
@@ -32,22 +49,39 @@ func TestNewIPList(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIPList_Contains(t *testing.T) {
|
||||
a := assert.NewAssertion(t)
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
list := NewIPList(IPListTypeDeny)
|
||||
var list = NewIPList(IPListTypeDeny)
|
||||
|
||||
for i := 0; i < 1_0000; i++ {
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
|
||||
}
|
||||
//list.RemoveIP("192.168.1.100")
|
||||
a.IsTrue(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1.100"))
|
||||
a.IsFalse(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.2.100"))
|
||||
{
|
||||
a.IsTrue(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1.100"))
|
||||
}
|
||||
{
|
||||
a.IsFalse(list.Contains(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.2.100"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPList_ContainsExpires(t *testing.T) {
|
||||
var list = NewIPList(IPListTypeDeny)
|
||||
|
||||
for i := 0; i < 1_0000; i++ {
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
|
||||
}
|
||||
// list.RemoveIP("192.168.1.100", 1, false)
|
||||
for _, ip := range []string{"192.168.1.100", "192.168.2.100"} {
|
||||
expiresAt, ok := list.ContainsExpires(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, ip)
|
||||
t.Log(ok, expiresAt, timeutil.FormatTime("Y-m-d H:i:s", expiresAt))
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkIPList_Add(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
list := NewIPList(IPListTypeDeny)
|
||||
var list = NewIPList(IPListTypeDeny)
|
||||
for i := 0; i < b.N; i++ {
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
|
||||
}
|
||||
@@ -57,7 +91,8 @@ func BenchmarkIPList_Add(b *testing.B) {
|
||||
func BenchmarkIPList_Has(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
list := NewIPList(IPListTypeDeny)
|
||||
var list = NewIPList(IPListTypeDeny)
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < 1_0000; i++ {
|
||||
list.Add(IPTypeAll, firewallconfigs.FirewallScopeGlobal, 1, "192.168.1."+strconv.Itoa(i), time.Now().Unix()+3600)
|
||||
|
||||
@@ -667,7 +667,7 @@ func (this *Rule) execFilter(value interface{}) interface{} {
|
||||
}
|
||||
value, goNext, err = filterInstance.Do(value, filter.Options)
|
||||
if err != nil {
|
||||
remotelogs.Println("WAF", "filter error: "+err.Error())
|
||||
remotelogs.Error("WAF", "filter error: "+err.Error())
|
||||
break
|
||||
}
|
||||
if !goNext {
|
||||
|
||||
@@ -136,19 +136,19 @@ func (this *RuleSet) ActionCodes() []string {
|
||||
return this.actionCodes
|
||||
}
|
||||
|
||||
func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Request, writer http.ResponseWriter) bool {
|
||||
func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Request, writer http.ResponseWriter) (continueRequest bool, goNextSet bool) {
|
||||
if len(waf.Mode) != 0 && waf.Mode != firewallconfigs.FirewallModeDefend {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
|
||||
// 先执行allow
|
||||
for _, instance := range this.actionInstances {
|
||||
if !instance.WillChange() {
|
||||
goNext := req.WAFOnAction(instance)
|
||||
if !goNext {
|
||||
return false
|
||||
continueRequest = req.WAFOnAction(instance)
|
||||
if !continueRequest {
|
||||
return false, false
|
||||
}
|
||||
instance.Perform(waf, group, this, req, writer)
|
||||
_, goNextSet = instance.Perform(waf, group, this, req, writer)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,15 +156,15 @@ func (this *RuleSet) PerformActions(waf *WAF, group *RuleGroup, req requests.Req
|
||||
for _, instance := range this.actionInstances {
|
||||
// 只执行第一个可能改变请求的动作,其余的都会被忽略
|
||||
if instance.WillChange() {
|
||||
goNext := req.WAFOnAction(instance)
|
||||
if !goNext {
|
||||
return false
|
||||
continueRequest = req.WAFOnAction(instance)
|
||||
if !continueRequest {
|
||||
return false, false
|
||||
}
|
||||
return instance.Perform(waf, group, this, req, writer)
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
return true, goNextSet
|
||||
}
|
||||
|
||||
func (this *RuleSet) MatchRequest(req requests.Request) (b bool, hasRequestBody bool, err error) {
|
||||
|
||||
@@ -241,7 +241,7 @@ func (this *WAF) MoveOutboundRuleGroup(fromIndex int, toIndex int) {
|
||||
this.Outbound = result
|
||||
}
|
||||
|
||||
func (this *WAF) MatchRequest(req requests.Request, writer http.ResponseWriter) (goNext bool, hasRequestBody bool, group *RuleGroup, set *RuleSet, err error) {
|
||||
func (this *WAF) MatchRequest(req requests.Request, writer http.ResponseWriter) (goNext bool, hasRequestBody bool, group *RuleGroup, sets *RuleSet, err error) {
|
||||
if !this.hasInboundRules {
|
||||
return true, hasRequestBody, nil, nil, nil
|
||||
}
|
||||
@@ -272,8 +272,10 @@ func (this *WAF) MatchRequest(req requests.Request, writer http.ResponseWriter)
|
||||
return true, hasRequestBody, nil, nil, err
|
||||
}
|
||||
if b {
|
||||
goNext = set.PerformActions(this, group, req, writer)
|
||||
return goNext, hasRequestBody, group, set, nil
|
||||
continueRequest, goNextSet := set.PerformActions(this, group, req, writer)
|
||||
if !goNextSet {
|
||||
return continueRequest, hasRequestBody, group, set, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, hasRequestBody, nil, nil, nil
|
||||
@@ -296,8 +298,10 @@ func (this *WAF) MatchResponse(req requests.Request, rawResp *http.Response, wri
|
||||
return true, hasRequestBody, nil, nil, err
|
||||
}
|
||||
if b {
|
||||
goNext = set.PerformActions(this, group, req, writer)
|
||||
return goNext, hasRequestBody, group, set, nil
|
||||
continueRequest, goNextSet := set.PerformActions(this, group, req, writer)
|
||||
if !goNextSet {
|
||||
return continueRequest, hasRequestBody, group, set, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, hasRequestBody, nil, nil, nil
|
||||
|
||||
Reference in New Issue
Block a user