自动升级一个SQL注入规则

go.mod

@@ -5,27 +5,23 @@ go 1.15
 replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
 
 require (
-	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183
 	github.com/andybalholm/brotli v1.0.4
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/go-acme/lego/v4 v4.5.2
-	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/golang/protobuf v1.5.2
-	github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24
+	github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
 	github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/lionsoul2014/ip2region v2.2.0-release+incompatible
 	github.com/mozillazg/go-pinyin v0.18.0
 	github.com/pkg/sftp v1.12.0
-	github.com/shirou/gopsutil v3.21.5+incompatible
-	github.com/tklauser/go-sysconf v0.3.6 // indirect
-	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
-	google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
-	google.golang.org/grpc v1.38.0
-	google.golang.org/protobuf v1.26.0
+	github.com/shirou/gopsutil/v3 v3.22.2 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
+	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
+	google.golang.org/grpc v1.45.0
+	google.golang.org/protobuf v1.27.1
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -43,8 +43,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87/go.mod h1:iGLljf5n9GjT6kc0HBvyI1nOKnGQbNB66VzSNbK5iks=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -52,6 +50,7 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbx
 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -78,6 +77,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/cloudflare-go v0.20.0/go.mod h1:sPWL/lIC6biLEdyGZwBQ1rGQKF1FhM7N60fuNiFdYTI=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -101,7 +104,8 @@ github.com/dnsimple/dnsimple-go v0.70.1/go.mod h1:F9WHww9cC76hrnwGFfAfrqdW99j3MO
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/exoscale/egoscale v0.67.0/go.mod h1:wi0myUxPsV8SdEtdJHQJxFLL/wEw9fiw9Gs1PWRkvkM=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -122,8 +126,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
-github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
@@ -132,8 +136,6 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
-github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -161,6 +163,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -175,6 +178,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
@@ -201,6 +207,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
@@ -231,11 +238,8 @@ github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhK
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/infobloxopen/infoblox-go-client v1.1.1/go.mod h1:BXiw7S2b9qJoM8MS40vfgCNB2NLHGusk1DtO16BD9zI=
 github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
-github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
-github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13 h1:HuEJ5xJfujW1Q6rNDhOu5LQXEBB2qLPah3jYslT8Gz4=
-github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
-github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24 h1:1cGulkD2SNJJRok5OKwyhP/Ddm+PgSWKOupn0cR36/A=
-github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
+github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
+github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
 github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
 github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
 github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
@@ -250,8 +254,6 @@ github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -277,12 +279,11 @@ github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c
 github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/linode/linodego v0.31.1/go.mod h1:BR0gVkCJffEdIGJSl6bHR80Ty+Uvg/2jkjmrWaFectM=
-github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
-github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
 github.com/liquidweb/go-lwApi v0.0.0-20190605172801-52a4864d2738/go.mod h1:0sYF9rMXb0vlG+4SzdiGMXHheCZxjguMq+Zb4S2BfBs=
 github.com/liquidweb/go-lwApi v0.0.5/go.mod h1:0sYF9rMXb0vlG+4SzdiGMXHheCZxjguMq+Zb4S2BfBs=
 github.com/liquidweb/liquidweb-cli v0.6.9/go.mod h1:cE1uvQ+x24NGUL75D0QagOFCG8Wdvmwu8aL9TLmA/eQ=
 github.com/liquidweb/liquidweb-go v1.6.3/go.mod h1:SuXXp+thr28LnjEw18AYtWwIbWMHSUiajPQs8T9c/Rc=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -322,7 +323,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
@@ -371,6 +371,7 @@ github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -390,14 +391,17 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS7IBEjKVSrjg=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
-github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
+github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -438,8 +442,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
 github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
+github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
+github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
 github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
+github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
+github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
@@ -456,13 +464,15 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -482,8 +492,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -507,7 +518,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -516,7 +526,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -551,8 +560,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -588,6 +599,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -611,6 +623,7 @@ golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -619,11 +632,17 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -631,8 +650,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -674,7 +694,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -714,9 +733,12 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
-google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
+google.golang.org/genproto v0.0.0-20220303160752-862486edd9cc h1:fb/ViRpv3ln/LvbqZtTpoOd1YQDNH12gaGZreoSFovE=
+google.golang.org/genproto v0.0.0-20220303160752-862486edd9cc/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
+google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -726,9 +748,12 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
+google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -739,8 +764,9 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/configs/api_config.go

@@ -2,8 +2,8 @@ package configs
 
 import (
 	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
-	"github.com/go-yaml/yaml"
 	"github.com/iwind/TeaGo/Tea"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"path/filepath"

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "0.4.2"
+	Version = "0.4.5"
 
 	ProductName   = "Edge API"
 	ProcessName   = "edge-api"
@@ -18,10 +18,13 @@ const (
 
 	// 其他节点版本号，用来检测是否有需要升级的节点
 
-	NodeVersion          = "0.4.2"
-	UserNodeVersion      = "0.3.1"
+	NodeVersion          = "0.4.5"
+	UserNodeVersion      = "0.3.2"
 	AuthorityNodeVersion = "0.0.2"
 	MonitorNodeVersion   = "0.0.3"
 	DNSNodeVersion       = "0.2.1"
 	ReportNodeVersion    = "0.1.0"
+
+	// SQLVersion SQL版本号
+	SQLVersion = "5"
 )

internal/db/models/db_node_initializer.go

@@ -29,8 +29,7 @@ type httpAccessLogDefinition struct {
 }
 
 // HTTP服务访问
-var httpAccessLogDAOMapping = map[int64]*HTTPAccessLogDAOWrapper{}    // dbNodeId => DAO
-var httpAccessLogTableMapping = map[string]*httpAccessLogDefinition{} // tableName_crc(dsn) => true
+var httpAccessLogDAOMapping = map[int64]*HTTPAccessLogDAOWrapper{} // dbNodeId => DAO
 
 // DNS服务访问
 var nsAccessLogDAOMapping = map[int64]*NSAccessLogDAOWrapper{} // dbNodeId => DAO
@@ -86,36 +85,6 @@ func randomNSAccessLogDAO() (dao *NSAccessLogDAOWrapper) {
 	return
 }
 
-// 检查表格是否存在
-func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, hasDomain bool, ok bool, err error) {
-	if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
-		err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
-		return
-	}
-
-	config, err := db.Config()
-	if err != nil {
-		return "", false, false, false, err
-	}
-
-	tableName = "edgeHTTPAccessLogs_" + day
-	cacheKey := tableName + "_" + fmt.Sprintf("%d", crc32.ChecksumIEEE([]byte(config.Dsn)))
-
-	accessLogLocker.RLock()
-	def, ok := httpAccessLogTableMapping[cacheKey]
-	accessLogLocker.RUnlock()
-	if ok {
-		return tableName, def.HasRemoteAddr, def.HasDomain, true, nil
-	}
-
-	def, err = findHTTPAccessLogTable(db, day, false)
-	if err != nil {
-		return tableName, false, false, false, err
-	}
-
-	return tableName, def.HasRemoteAddr, def.HasDomain, def.Exists, nil
-}
-
 func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) {
 	if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
 		err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
@@ -145,75 +114,6 @@ func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool
 	return tableName, utils.ContainsStringInsensitive(tableNames, tableName), nil
 }
 
-// 根据日期获取表名
-func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
-	config, err := db.Config()
-	if err != nil {
-		return nil, err
-	}
-
-	tableName := "edgeHTTPAccessLogs_" + day
-	cacheKey := tableName + "_" + fmt.Sprintf("%d", crc32.ChecksumIEEE([]byte(config.Dsn)))
-
-	if !force {
-		accessLogLocker.RLock()
-		definition, ok := httpAccessLogTableMapping[cacheKey]
-		accessLogLocker.RUnlock()
-		if ok {
-			return definition, nil
-		}
-	}
-
-	tableNames, err := db.TableNames()
-	if err != nil {
-		return nil, err
-	}
-
-	if utils.ContainsStringInsensitive(tableNames, tableName) {
-		table, err := db.FindTable(tableName)
-		if err != nil {
-			return nil, err
-		}
-
-		accessLogLocker.Lock()
-		var definition = &httpAccessLogDefinition{
-			Name:          tableName,
-			HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil,
-			HasDomain:     table.FindFieldWithName("domain") != nil,
-			Exists:        true,
-		}
-		httpAccessLogTableMapping[cacheKey] = definition
-		accessLogLocker.Unlock()
-		return definition, nil
-	}
-
-	if !force {
-		return &httpAccessLogDefinition{
-			Name:          tableName,
-			HasRemoteAddr: true,
-			HasDomain:     true,
-			Exists:        false,
-		}, nil
-	}
-
-	// 创建表格
-	_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n  `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n  `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n  `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n  `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n  `content` json DEFAULT NULL COMMENT '日志内容',\n  `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n  `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n  `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n  `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n  `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n  `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',\n  `domain` varchar(128) DEFAULT NULL COMMENT '域名',\n  `requestBody` mediumblob COMMENT '请求内容',\n  `responseBody` mediumblob COMMENT '响应内容',\n  PRIMARY KEY (`id`),\n  KEY `serverId` (`serverId`),\n  KEY `nodeId` (`nodeId`),\n  KEY `serverId_status` (`serverId`,`status`),\n  KEY `requestId` (`requestId`),\n  KEY `firewallPolicyId` (`firewallPolicyId`),\n  KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n  KEY `firewallRuleSetId` (`firewallRuleSetId`),\n  KEY `firewallRuleId` (`firewallRuleId`),\n  KEY `remoteAddr` (`remoteAddr`),\n  KEY `domain` (`domain`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
-	if err != nil {
-		return nil, err
-	}
-
-	accessLogLocker.Lock()
-	var definition = &httpAccessLogDefinition{
-		Name:          tableName,
-		HasRemoteAddr: true,
-		Exists:        true,
-	}
-	httpAccessLogTableMapping[cacheKey] = definition
-	accessLogLocker.Unlock()
-
-	return definition, nil
-}
-
 func findNSAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
 	config, err := db.Config()
 	if err != nil {
@@ -351,22 +251,17 @@ func (this *DBNodeInitializer) loop() error {
 			// 检查表是否存在
 			// httpAccessLog
 			{
-				tableDef, err := findHTTPAccessLogTable(db, timeutil.Format("Ymd"), true)
+				tableDef, err := SharedHTTPAccessLogManager.FindTable(db, timeutil.Format("Ymd"), true)
 				if err != nil {
-					if !strings.Contains(err.Error(), "1050") { // 非表格已存在错误
-						remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
+					remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
 
-						// 创建节点日志
-						createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix(), "", nil)
-						if createLogErr != nil {
-							remotelogs.Error("NODE_LOG", createLogErr.Error())
-						}
-
-						continue
-					} else {
-						err = nil
-						continue
+					// 创建节点日志
+					createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix(), "", nil)
+					if createLogErr != nil {
+						remotelogs.Error("NODE_LOG", createLogErr.Error())
 					}
+
+					continue
 				}
 
 				daoObject := dbs.DAOObject{

internal/db/models/db_node_initializer_test.go

@@ -1,9 +1,7 @@
 package models
 
 import (
-	"runtime"
 	"testing"
-	"time"
 )
 
 func TestDBNodeInitializer_loop(t *testing.T) {
@@ -14,32 +12,3 @@ func TestDBNodeInitializer_loop(t *testing.T) {
 	}
 	t.Log(len(accessLogDBMapping), len(httpAccessLogDAOMapping))
 }
-
-func TestFindAccessLogTable(t *testing.T) {
-	before := time.Now()
-	db := SharedHTTPAccessLogDAO.Instance
-	tableName, err := findHTTPAccessLogTable(db, "20201010", false)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(tableName)
-	t.Log(time.Since(before).Seconds()*1000, "ms")
-
-	before = time.Now()
-	tableName, err = findHTTPAccessLogTable(db, "20201010", false)
-
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(tableName)
-	t.Log(time.Since(before).Seconds()*1000, "ms")
-}
-
-func BenchmarkFindAccessLogTable(b *testing.B) {
-	db := SharedHTTPAccessLogDAO.Instance
-
-	runtime.GOMAXPROCS(1)
-	for i := 0; i < b.N; i++ {
-		_, _ = findHTTPAccessLogTable(db, "20201010", false)
-	}
-}

internal/db/models/http_access_log_dao.go

@@ -34,13 +34,25 @@ type HTTPAccessLogDAO dbs.DAO
 var SharedHTTPAccessLogDAO *HTTPAccessLogDAO
 
 // 队列
-var oldAccessLogQueue = make(chan *pb.HTTPAccessLog)
-var accessLogQueue = make(chan *pb.HTTPAccessLog, 10_000)
-var accessLogQueueMaxLength = 100_000
-var accessLogQueuePercent = 100      // 0-100
-var accessLogCountPerSecond = 10_000 // 0 表示不限制
-var accessLogConfigJSON = []byte{}
-var accessLogQueueChanged = make(chan zero.Zero, 1)
+var (
+	oldAccessLogQueue       = make(chan *pb.HTTPAccessLog)
+	accessLogQueue          = make(chan *pb.HTTPAccessLog, 10_000)
+	accessLogQueueMaxLength = 100_000
+	accessLogQueuePercent   = 100    // 0-100
+	accessLogCountPerSecond = 10_000 // 0 表示不限制
+	accessLogConfigJSON     = []byte{}
+	accessLogQueueChanged   = make(chan zero.Zero, 1)
+
+	accessLogEnableAutoPartial       = true    // 是否启用自动分表
+	accessLogRowsPerTable      int64 = 500_000 // 自动分表的单表最大值
+)
+
+type accessLogTableQuery struct {
+	daoWrapper         *HTTPAccessLogDAOWrapper
+	name               string
+	hasRemoteAddrField bool
+	hasDomainField     bool
+}
 
 func init() {
 	dbs.OnReady(func() {
@@ -120,7 +132,7 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogs(tx *dbs.Tx, accessLogs []*pb.
 
 // DumpAccessLogsFromQueue 从队列导入访问日志
 func (this *HTTPAccessLogDAO) DumpAccessLogsFromQueue(tx *dbs.Tx, size int) error {
-	dao := randomHTTPAccessLogDAO()
+	var dao = randomHTTPAccessLogDAO()
 	if dao == nil {
 		dao = &HTTPAccessLogDAOWrapper{
 			DAO:    SharedHTTPAccessLogDAO,
@@ -168,8 +180,8 @@ Loop:
 
 // CreateHTTPAccessLog 写入单条访问日志
 func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLogDAO, accessLog *pb.HTTPAccessLog) error {
-	day := timeutil.Format("Ymd", time.Unix(accessLog.Timestamp, 0))
-	tableDef, err := findHTTPAccessLogTable(dao.Instance, day, false)
+	var day = timeutil.Format("Ymd", time.Unix(accessLog.Timestamp, 0))
+	tableDef, err := SharedHTTPAccessLogManager.FindTable(dao.Instance, day, true)
 	if err != nil {
 		return err
 	}
@@ -203,27 +215,17 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLog
 	}
 	fields["content"] = content
 
-	_, err = dao.Query(tx).
+	var lastId int64
+	lastId, err = dao.Query(tx).
 		Table(tableDef.Name).
 		Sets(fields).
 		Insert()
 	if err != nil {
-		// 是否为 Error 1146: Table 'xxx.xxx' doesn't exist  如果是，则创建表之后重试
-		if strings.Contains(err.Error(), "1146") {
-			tableDef, err = findHTTPAccessLogTable(dao.Instance, day, true)
-			if err != nil {
-				return err
-			}
-			_, err = dao.Query(tx).
-				Table(tableDef.Name).
-				Sets(fields).
-				Insert()
-			if err != nil {
-				return err
-			}
-		} else {
-			remotelogs.Error("HTTP_ACCESS_LOG", err.Error())
-		}
+		return err
+	}
+
+	if accessLogEnableAutoPartial && accessLogRowsPerTable > 0 && lastId%accessLogRowsPerTable == 0 {
+		SharedHTTPAccessLogManager.ResetTable(dao.Instance, day)
 	}
 
 	return nil
@@ -233,6 +235,8 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLog
 func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
 	size int64,
 	day string,
+	hourFrom string,
+	hourTo string,
 	clusterId int64,
 	nodeId int64,
 	serverId int64,
@@ -255,18 +259,36 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
 		size = 1000
 	}
 
-	result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
+	result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	if err != nil || int64(len(result)) < size {
 		return
 	}
 
-	moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
+	moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	hasMore = len(moreResult) > 0
 	return
 }
 
 // 读取往前的单页访问日志
-func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, clusterId int64, nodeId int64, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string, ip string, domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
+func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
+	lastRequestId string,
+	size int64,
+	day string,
+	hourFrom string,
+	hourTo string,
+	clusterId int64,
+	nodeId int64,
+	serverId int64,
+	reverse bool,
+	hasError bool,
+	firewallPolicyId int64,
+	firewallRuleGroupId int64,
+	firewallRuleSetId int64,
+	hasFirewallPolicy bool,
+	userId int64,
+	keyword string,
+	ip string,
+	domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
 	if size <= 0 {
 		return nil, lastRequestId, nil
 	}
@@ -296,42 +318,57 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 		}}
 	}
 
-	locker := sync.Mutex{}
+	// 查询某个集群下的节点
+	var nodeIds = []int64{}
+	if clusterId > 0 {
+		nodeIds, err = SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
+		if err != nil {
+			remotelogs.Error("DBNODE", err.Error())
+			return
+		}
+		sort.Slice(nodeIds, func(i, j int) bool {
+			return nodeIds[i] < nodeIds[j]
+		})
+	}
 
-	count := len(daoList)
-	wg := &sync.WaitGroup{}
-	wg.Add(count)
+	// 准备查询
+	var tableQueries = []*accessLogTableQuery{}
 	for _, daoWrapper := range daoList {
-		go func(daoWrapper *HTTPAccessLogDAOWrapper) {
+		var instance = daoWrapper.DAO.Instance
+		tableDefs, err := SharedHTTPAccessLogManager.FindTables(instance, day)
+		if err != nil {
+			return nil, "", err
+		}
+		for _, def := range tableDefs {
+			tableQueries = append(tableQueries, &accessLogTableQuery{
+				daoWrapper:         daoWrapper,
+				name:               def.Name,
+				hasRemoteAddrField: def.HasRemoteAddr,
+				hasDomainField:     def.HasDomain,
+			})
+		}
+	}
+
+	var locker = sync.Mutex{}
+
+	var statusPrefixReg = regexp.MustCompile(`status:\s*(\d{3})\b`)
+	var statusRangeReg = regexp.MustCompile(`status:\s*(\d{3})-(\d{3})\b`)
+
+	var count = len(tableQueries)
+	var wg = &sync.WaitGroup{}
+	wg.Add(count)
+	for _, tableQuery := range tableQueries {
+		go func(tableQuery *accessLogTableQuery) {
 			defer wg.Done()
 
-			dao := daoWrapper.DAO
-
-			tableName, hasRemoteAddrField, hasDomainField, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
-			if err != nil {
-				logs.Println("[DB_NODE]" + err.Error())
-				return
-			}
-			if !exists {
-				// 表格不存在则跳过
-				return
-			}
-
-			query := dao.Query(tx)
+			var dao = tableQuery.daoWrapper.DAO
+			var query = dao.Query(tx)
 
 			// 条件
 			if nodeId > 0 {
 				query.Attr("nodeId", nodeId)
 			} else if clusterId > 0 {
-				nodeIds, err := SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
-				if err != nil {
-					remotelogs.Error("DBNODE", err.Error())
-					return
-				}
 				if len(nodeIds) > 0 {
-					sort.Slice(nodeIds, func(i, j int) bool {
-						return nodeIds[i] < nodeIds[j]
-					})
 					var nodeIdStrings = []string{}
 					for _, subNodeId := range nodeIds {
 						nodeIdStrings = append(nodeIdStrings, types.String(subNodeId))
@@ -370,7 +407,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 			// keyword
 			if len(ip) > 0 {
 				// TODO 支持IP范围
-				if hasRemoteAddrField {
+				if tableQuery.hasRemoteAddrField {
 					// IP格式
 					if strings.Contains(ip, ",") || strings.Contains(ip, "-") {
 						rangeConfig, err := shared.ParseIPRange(ip)
@@ -380,6 +417,9 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 							}
 						}
 					} else {
+						// 去掉IPv6的[]
+						ip = strings.Trim(ip, "[]")
+
 						query.Attr("remoteAddr", ip)
 						query.UseIndex("remoteAddr")
 					}
@@ -389,7 +429,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 				}
 			}
 			if len(domain) > 0 {
-				if hasDomainField {
+				if tableQuery.hasDomainField {
 					if strings.Contains(domain, "*") {
 						domain = strings.ReplaceAll(domain, "*", "%")
 						domain = regexp.MustCompile(`[^a-zA-Z0-9-.%]`).ReplaceAllString(domain, "")
@@ -404,11 +444,12 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 						Param("host1", domain)
 				}
 			}
+
 			if len(keyword) > 0 {
 				// remoteAddr
-				if hasRemoteAddrField && net.ParseIP(keyword) != nil {
+				if tableQuery.hasRemoteAddrField && net.ParseIP(keyword) != nil {
 					query.Attr("remoteAddr", keyword)
-				} else if hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
+				} else if tableQuery.hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
 					keyword = keyword[3:]
 					pieces := strings.SplitN(keyword, ",", 2)
 					if len(pieces) == 1 || len(pieces[1]) == 0 {
@@ -416,12 +457,21 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 					} else {
 						query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
 					}
+				} else if statusRangeReg.MatchString(keyword) {
+					var matches = statusRangeReg.FindStringSubmatch(keyword)
+					query.Between("status", types.Int(matches[1]), types.Int(matches[2]))
+
+					// TODO 处理剩余的关键词
+				} else if statusPrefixReg.MatchString(keyword) {
+					var matches = statusPrefixReg.FindStringSubmatch(keyword)
+					query.Attr("status", matches[1])
+					// TODO 处理剩余的关键词
 				} else {
 					if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
 						keyword = keyword[3:]
 					}
 
-					useOriginKeyword := false
+					var useOriginKeyword = false
 
 					where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword OR JSON_EXTRACT(content, '$.userAgent') LIKE :keyword"
 
@@ -447,13 +497,13 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 
 					// 响应状态码
 					if regexp.MustCompile(`^\d{3}$`).MatchString(keyword) {
-						where += " OR JSON_EXTRACT(content, '$.status')=:intKeyword"
+						where += " OR status=:intKeyword"
 						query.Param("intKeyword", types.Int(keyword))
 					}
 
 					if regexp.MustCompile(`^\d{3}-\d{3}$`).MatchString(keyword) {
 						pieces := strings.Split(keyword, "-")
-						where += " OR JSON_EXTRACT(content, '$.status') BETWEEN :intKeyword1 AND :intKeyword2"
+						where += " OR status BETWEEN :intKeyword1 AND :intKeyword2"
 						query.Param("intKeyword1", types.Int(pieces[0]))
 						query.Param("intKeyword2", types.Int(pieces[1]))
 					}
@@ -471,6 +521,20 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 				}
 			}
 
+			// hourFrom - hourTo
+			if len(hourFrom) > 0 && len(hourTo) > 0 {
+				var hourFromInt = types.Int(hourFrom)
+				var hourToInt = types.Int(hourTo)
+				if hourFromInt >= 0 && hourFromInt <= 23 && hourToInt >= hourFromInt && hourToInt <= 23 {
+					var y = types.Int(day[:4])
+					var m = types.Int(day[4:6])
+					var d = types.Int(day[6:])
+					var timeFrom = time.Date(y, time.Month(m), d, hourFromInt, 0, 0, 0, time.Local)
+					var timeTo = time.Date(y, time.Month(m), d, hourToInt, 59, 59, 0, time.Local)
+					query.Between("createdAt", timeFrom.Unix(), timeTo.Unix())
+				}
+			}
+
 			// offset
 			if len(lastRequestId) > 0 {
 				if !reverse {
@@ -490,20 +554,21 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 
 			// 开始查询
 			ones, err := query.
-				Table(tableName).
+				Table(tableQuery.name).
 				Limit(size).
 				FindAll()
 			if err != nil {
 				logs.Println("[DB_NODE]" + err.Error())
 				return
 			}
+
 			locker.Lock()
 			for _, one := range ones {
 				accessLog := one.(*HTTPAccessLog)
 				result = append(result, accessLog)
 			}
 			locker.Unlock()
-		}(daoWrapper)
+		}(tableQuery)
 	}
 	wg.Wait()
 
@@ -524,7 +589,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
 		result = result[:size]
 	}
 
-	requestId := result[len(result)-1].RequestId
+	var requestId = result[len(result)-1].RequestId
 	if reverse {
 		lists.Reverse(result)
 	}
@@ -556,28 +621,36 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s
 		}}
 	}
 
-	count := len(daoList)
-	wg := &sync.WaitGroup{}
+	// 准备查询
+	var day = timeutil.FormatTime("Ymd", types.Int64(requestId[:10]))
+	var tableQueries = []*accessLogTableQuery{}
+	for _, daoWrapper := range daoList {
+		var instance = daoWrapper.DAO.Instance
+		tableDefs, err := SharedHTTPAccessLogManager.FindTables(instance, day)
+		if err != nil {
+			return nil, err
+		}
+		for _, def := range tableDefs {
+			tableQueries = append(tableQueries, &accessLogTableQuery{
+				daoWrapper:         daoWrapper,
+				name:               def.Name,
+				hasRemoteAddrField: def.HasRemoteAddr,
+				hasDomainField:     def.HasDomain,
+			})
+		}
+	}
+
+	var count = len(tableQueries)
+	var wg = &sync.WaitGroup{}
 	wg.Add(count)
 	var result *HTTPAccessLog = nil
-	day := timeutil.FormatTime("Ymd", types.Int64(requestId[:10]))
-	for _, daoWrapper := range daoList {
-		go func(daoWrapper *HTTPAccessLogDAOWrapper) {
+	for _, tableQuery := range tableQueries {
+		go func(tableQuery *accessLogTableQuery) {
 			defer wg.Done()
 
-			dao := daoWrapper.DAO
-
-			tableName, _, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
-			if err != nil {
-				logs.Println("[DB_NODE]" + err.Error())
-				return
-			}
-			if !exists {
-				return
-			}
-
+			var dao = tableQuery.daoWrapper.DAO
 			one, err := dao.Query(tx).
-				Table(tableName).
+				Table(tableQuery.name).
 				Attr("requestId", requestId).
 				Find()
 			if err != nil {
@@ -587,7 +660,7 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s
 			if one != nil {
 				result = one.(*HTTPAccessLog)
 			}
-		}(daoWrapper)
+		}(tableQuery)
 	}
 	wg.Wait()
 	return result, nil
@@ -623,11 +696,18 @@ func (this *HTTPAccessLogDAO) SetupQueue() {
 		config.MaxLength = 100_000
 	}
 
+	accessLogEnableAutoPartial = config.EnableAutoPartial
+	if config.RowsPerTable > 0 {
+		accessLogRowsPerTable = config.RowsPerTable
+	}
+
 	if accessLogQueueMaxLength != config.MaxLength {
 		accessLogQueueMaxLength = config.MaxLength
 		oldAccessLogQueue = accessLogQueue
 		accessLogQueue = make(chan *pb.HTTPAccessLog, config.MaxLength)
 	}
 
-	remotelogs.Println("HTTP_ACCESS_LOG_QUEUE", "change queue max length: "+types.String(config.MaxLength)+", percent: "+types.String(config.Percent)+", countPerSecond: "+types.String(config.CountPerSecond))
+	if Tea.IsTesting() {
+		remotelogs.Println("HTTP_ACCESS_LOG_QUEUE", "change queue "+string(configJSON))
+	}
 }

internal/db/models/http_access_log_dao_test.go

@@ -53,7 +53,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -80,7 +80,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
 	times := 0 // 防止循环次数太多
 	for {
 		before := time.Now()
-		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 		cost := time.Since(before).Seconds()
 		if err != nil {
 			t.Fatal(err)
@@ -111,7 +111,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
 	}
 
 	before := time.Now()
-	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
+	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
 	cost := time.Since(before).Seconds()
 	if err != nil {
 		t.Fatal(err)
@@ -136,7 +136,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
 	times := 0 // 防止循环次数太多
 	for {
 		before := time.Now()
-		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 		cost := time.Since(before).Seconds()
 		if err != nil {
 			t.Fatal(err)

internal/db/models/http_access_log_manager.go

@@ -0,0 +1,297 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package models
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/types"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+)
+
+// 访问日志的两个表格形式
+var accessLogTableMainReg = regexp.MustCompile(`_(\d{8})$`)
+var accessLogTablePartialReg = regexp.MustCompile(`_(\d{8})_(\d{4})$`)
+
+var SharedHTTPAccessLogManager = NewHTTPAccessLogManager()
+
+type HTTPAccessLogManager struct {
+	currentTableMapping map[string]*httpAccessLogDefinition // dsn => def
+
+	locker sync.Mutex
+}
+
+func NewHTTPAccessLogManager() *HTTPAccessLogManager {
+	return &HTTPAccessLogManager{
+		currentTableMapping: map[string]*httpAccessLogDefinition{},
+	}
+}
+
+// FindTableNames 读取数据库中某日所有日志表名称
+func (this *HTTPAccessLogManager) FindTableNames(db *dbs.DB, day string) ([]string, error) {
+	var results = []string{}
+
+	// 需要防止用户设置了表名自动小写
+	for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
+		ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
+		if err != nil {
+			return nil, errors.New("query table names error: " + err.Error())
+		}
+
+		var columnName = columnNames[0]
+
+		for _, one := range ones {
+			var tableName = one[columnName].(string)
+
+			if lists.ContainsString(results, tableName) {
+				continue
+			}
+
+			if accessLogTableMainReg.MatchString(tableName) || accessLogTablePartialReg.MatchString(tableName) {
+				results = append(results, tableName)
+			}
+		}
+	}
+
+	// 排序
+	sort.Strings(results)
+
+	return results, nil
+}
+
+// FindTables 读取数据库中某日所有日志表
+func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAccessLogDefinition, error) {
+	var results = []*httpAccessLogDefinition{}
+	var tableNames = []string{}
+
+	// 需要防止用户设置了表名自动小写
+	for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
+		ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
+		if err != nil {
+			return nil, errors.New("query table names error: " + err.Error())
+		}
+
+		var columnName = columnNames[0]
+
+		for _, one := range ones {
+			var tableName = one[columnName].(string)
+
+			if lists.ContainsString(tableNames, tableName) {
+				continue
+			}
+
+			if accessLogTableMainReg.MatchString(tableName) {
+				tableNames = append(tableNames, tableName)
+
+				hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, tableName)
+				if err != nil {
+					return nil, err
+				}
+
+				results = append(results, &httpAccessLogDefinition{
+					Name:          tableName,
+					HasRemoteAddr: hasRemoteAddrField,
+					HasDomain:     hasDomainField,
+					Exists:        true,
+				})
+			} else if accessLogTablePartialReg.MatchString(tableName) {
+				tableNames = append(tableNames, tableName)
+
+				results = append(results, &httpAccessLogDefinition{
+					Name:          tableName,
+					HasRemoteAddr: true,
+					HasDomain:     true,
+					Exists:        true,
+				})
+			}
+		}
+	}
+
+	// 排序
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].Name < results[j].Name
+	})
+
+	return results, nil
+}
+
+// FindTable 根据日期获取表名
+// 表名组成
+//   - PREFIX_DAY
+//   - PREFIX_DAY_0001
+func (this *HTTPAccessLogManager) FindTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	config, err := db.Config()
+	if err != nil {
+		return nil, err
+	}
+	var cacheKey = config.Dsn
+	def, ok := this.currentTableMapping[cacheKey]
+	if ok {
+		return def, nil
+	}
+
+	def, err = this.findTableWithoutCache(db, day, force)
+	if err != nil {
+		return nil, err
+	}
+
+	this.currentTableMapping[cacheKey] = def
+	return def, nil
+}
+
+// CreateTable 创建访问日志表格
+func (this *HTTPAccessLogManager) CreateTable(db *dbs.DB, tableName string) error {
+	_, err := db.Exec("CREATE TABLE `" + tableName + "` (\n  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n  `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n  `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n  `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n  `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n  `content` json DEFAULT NULL COMMENT '日志内容',\n  `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n  `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n  `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n  `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n  `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n  `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',\n  `domain` varchar(128) DEFAULT NULL COMMENT '域名',\n  `requestBody` mediumblob COMMENT '请求内容',\n  `responseBody` mediumblob COMMENT '响应内容',\n  PRIMARY KEY (`id`),\n  KEY `serverId` (`serverId`),\n  KEY `nodeId` (`nodeId`),\n  KEY `serverId_status` (`serverId`,`status`),\n  KEY `requestId` (`requestId`),\n  KEY `firewallPolicyId` (`firewallPolicyId`),\n  KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n  KEY `firewallRuleSetId` (`firewallRuleSetId`),\n  KEY `firewallRuleId` (`firewallRuleId`),\n  KEY `remoteAddr` (`remoteAddr`),\n  KEY `domain` (`domain`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
+	if err != nil {
+		// 快速判断错误方法
+		mysqlErr, ok := err.(*mysql.MySQLError)
+		if ok && mysqlErr.Number == 1050 { // Error 1050: Table 'xxx' already exists
+			return nil
+		}
+
+		// 防止二次包装过程中错误丢失的保底错误判断方法
+		if strings.Contains(err.Error(), "Error 1050") {
+			return nil
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+// ResetTable 清除某个数据库表名缓存
+func (this *HTTPAccessLogManager) ResetTable(db *dbs.DB, day string) {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	config, err := db.Config()
+	if err != nil {
+		return
+	}
+	delete(this.currentTableMapping, config.Dsn)
+}
+
+// 查找某个表格
+func (this *HTTPAccessLogManager) findTableWithoutCache(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
+	tableNames, err := this.FindTableNames(db, day)
+	if err != nil {
+		return nil, err
+	}
+
+	var prefix = "edgeHTTPAccessLogs_" + day
+
+	if len(tableNames) == 0 {
+		if force {
+			err := this.CreateTable(db, prefix)
+			if err != nil {
+				return nil, err
+			}
+
+			return &httpAccessLogDefinition{
+				Name:          prefix,
+				HasRemoteAddr: true,
+				HasDomain:     true,
+				Exists:        true,
+			}, nil
+		}
+
+		return &httpAccessLogDefinition{
+			Name:          prefix,
+			HasRemoteAddr: true,
+			HasDomain:     true,
+			Exists:        false,
+		}, nil
+	}
+
+	var lastTableName = tableNames[len(tableNames)-1]
+	if !force || !accessLogEnableAutoPartial || accessLogRowsPerTable <= 0 {
+		hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, lastTableName)
+		if err != nil {
+			return nil, err
+		}
+		return &httpAccessLogDefinition{
+			Name:          lastTableName,
+			HasRemoteAddr: hasRemoteAddrField,
+			HasDomain:     hasDomainField,
+			Exists:        true,
+		}, nil
+	}
+
+	// 检查是否生成下个分表
+	lastId, err := db.FindCol(0, "SELECT id FROM "+lastTableName+" ORDER BY id DESC LIMIT 1")
+	if err != nil {
+		return nil, err
+	}
+
+	if lastId != nil {
+		var lastInt64Id = types.Int64(lastId)
+		if accessLogRowsPerTable > 0 && lastInt64Id >= accessLogRowsPerTable {
+			// create next partial table
+			var nextTableName = ""
+			if accessLogTableMainReg.MatchString(lastTableName) {
+				nextTableName = prefix + "_0001"
+			} else if accessLogTablePartialReg.MatchString(lastTableName) {
+				var matches = accessLogTablePartialReg.FindStringSubmatch(lastTableName)
+				if len(matches) < 3 {
+					return nil, errors.New("fatal error: invalid 'accessLogTablePartialReg'")
+				}
+				var lastPartial = matches[2]
+				nextTableName = prefix + "_" + fmt.Sprintf("%04d", types.Int(lastPartial)+1)
+			} else {
+				nextTableName = prefix + "_0001"
+			}
+
+			err = this.CreateTable(db, nextTableName)
+			if err != nil {
+				return nil, err
+			}
+
+			return &httpAccessLogDefinition{
+				Name:          nextTableName,
+				HasRemoteAddr: true,
+				HasDomain:     true,
+				Exists:        true,
+			}, nil
+		}
+	}
+
+	// 检查字段
+	hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, lastTableName)
+	if err != nil {
+		return nil, err
+	}
+	return &httpAccessLogDefinition{
+		Name:          lastTableName,
+		HasRemoteAddr: hasRemoteAddrField,
+		HasDomain:     hasDomainField,
+		Exists:        true,
+	}, nil
+}
+
+// TODO 考虑缓存检查结果
+func (this *HTTPAccessLogManager) checkTableFields(db *dbs.DB, tableName string) (hasRemoteAddrField bool, hasDomainField bool, err error) {
+	fields, _, err := db.FindOnes("SHOW FIELDS FROM " + tableName)
+	if err != nil {
+		return false, false, err
+	}
+	for _, field := range fields {
+		var fieldName = field.GetString("Field")
+		if strings.ToLower(fieldName) == strings.ToLower("remoteAddr") {
+			hasRemoteAddrField = true
+		}
+		if strings.ToLower(fieldName) == "domain" {
+			hasDomainField = true
+		}
+	}
+	return
+}

internal/db/models/http_access_log_manager_test.go

@@ -0,0 +1,148 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package models_test
+
+import (
+	"encoding/json"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+	"time"
+)
+
+func TestNewHTTPAccessLogManager(t *testing.T) {
+	var config = &dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+		Connections: struct {
+			Pool         int           `yaml:"pool"`
+			Max          int           `yaml:"max"`
+			Life         string        `yaml:"life"`
+			LifeDuration time.Duration `yaml:",omitempty"`
+		}{},
+		Models: struct {
+			Package string `yaml:"package"`
+		}{},
+	}
+
+	db, err := dbs.NewInstanceFromConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var manager = models.SharedHTTPAccessLogManager
+	err = manager.CreateTable(db, "accessLog_1")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestHTTPAccessLogManager_FindTableNames(t *testing.T) {
+	var config = &dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+		Connections: struct {
+			Pool         int           `yaml:"pool"`
+			Max          int           `yaml:"max"`
+			Life         string        `yaml:"life"`
+			LifeDuration time.Duration `yaml:",omitempty"`
+		}{},
+		Models: struct {
+			Package string `yaml:"package"`
+		}{},
+	}
+
+	db, err := dbs.NewInstanceFromConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 3; i++ {
+		var before = time.Now()
+		tables, err := models.SharedHTTPAccessLogManager.FindTables(db, "20220306")
+		if err != nil {
+			t.Fatal(err)
+		}
+		data, err := json.Marshal(tables)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(string(data))
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}
+}
+
+
+func TestHTTPAccessLogManager_FindTables(t *testing.T) {
+	var config = &dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+		Connections: struct {
+			Pool         int           `yaml:"pool"`
+			Max          int           `yaml:"max"`
+			Life         string        `yaml:"life"`
+			LifeDuration time.Duration `yaml:",omitempty"`
+		}{},
+		Models: struct {
+			Package string `yaml:"package"`
+		}{},
+	}
+
+	db, err := dbs.NewInstanceFromConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 3; i++ {
+		var before = time.Now()
+		tables, err := models.SharedHTTPAccessLogManager.FindTables(db, "20220306")
+		if err != nil {
+			t.Fatal(err)
+		}
+		data, err := json.Marshal(tables)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(string(data))
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}
+}
+
+func TestHTTPAccessLogManager_FindTable(t *testing.T) {
+	var config = &dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+		Connections: struct {
+			Pool         int           `yaml:"pool"`
+			Max          int           `yaml:"max"`
+			Life         string        `yaml:"life"`
+			LifeDuration time.Duration `yaml:",omitempty"`
+		}{},
+		Models: struct {
+			Package string `yaml:"package"`
+		}{},
+	}
+
+	db, err := dbs.NewInstanceFromConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 3; i++ {
+		var before = time.Now()
+		tableDef, err := models.SharedHTTPAccessLogManager.FindTable(db, "20220306", false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		data, err := json.Marshal(tableDef)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log(string(data))
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}
+}

internal/db/models/http_cache_policy_dao.go

@@ -95,7 +95,7 @@ func (this *HTTPCachePolicyDAO) FindAllEnabledCachePolicies(tx *dbs.Tx) (result
 }
 
 // CreateCachePolicy 创建缓存策略
-func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte) (int64, error) {
+func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) (int64, error) {
 	op := NewHTTPCachePolicyOperator()
 	op.State = HTTPCachePolicyStateEnabled
 	op.IsOn = isOn
@@ -112,6 +112,7 @@ func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name st
 	if len(storageOptionsJSON) > 0 {
 		op.Options = storageOptionsJSON
 	}
+	op.SyncCompressionCache = syncCompressionCache
 
 	// 默认的缓存条件
 	cacheRef := &serverconfigs.HTTPCacheRef{
@@ -182,13 +183,19 @@ func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string
 
 	var storageOptions = &serverconfigs.HTTPFileCacheStorage{
 		Dir: "/opt/cache",
+		MemoryPolicy: &serverconfigs.HTTPCachePolicy{
+			Capacity: &shared.SizeCapacity{
+				Count: 1,
+				Unit:  shared.SizeCapacityUnitGB,
+			},
+		},
 	}
 	storageOptionsJSON, err := json.Marshal(storageOptions)
 	if err != nil {
 		return 0, err
 	}
 
-	policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, 0, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON)
+	policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, 0, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON, false)
 	if err != nil {
 		return 0, err
 	}
@@ -196,7 +203,7 @@ func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string
 }
 
 // UpdateCachePolicy 修改缓存策略
-func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte) error {
+func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) error {
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
@@ -217,6 +224,7 @@ func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, is
 	if len(storageOptionsJSON) > 0 {
 		op.Options = storageOptionsJSON
 	}
+	op.SyncCompressionCache = syncCompressionCache
 	err := this.Save(tx, op)
 	if err != nil {
 		return err
@@ -247,6 +255,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
 	config.IsOn = policy.IsOn == 1
 	config.Name = policy.Name
 	config.Description = policy.Description
+	config.SyncCompressionCache = policy.SyncCompressionCache == 1
 
 	// capacity
 	if IsNotNull(policy.Capacity) {
@@ -300,7 +309,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
 }
 
 // CountAllEnabledHTTPCachePolicies 计算可用缓存策略数量
-func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string) (int64, error) {
+func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string, storageType string) (int64, error) {
 	query := this.Query(tx).
 		State(HTTPCachePolicyStateEnabled)
 	if clusterId > 0 {
@@ -311,11 +320,14 @@ func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clu
 		query.Where("(name LIKE :keyword)").
 			Param("keyword", "%"+keyword+"%")
 	}
+	if len(storageType) > 0 {
+		query.Attr("type", storageType)
+	}
 	return query.Count()
 }
 
 // ListEnabledHTTPCachePolicies 列出单页的缓存策略
-func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string, offset int64, size int64) ([]*serverconfigs.HTTPCachePolicy, error) {
+func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string, storageType string, offset int64, size int64) ([]*serverconfigs.HTTPCachePolicy, error) {
 	query := this.Query(tx).
 		State(HTTPCachePolicyStateEnabled)
 	if clusterId > 0 {
@@ -326,6 +338,9 @@ func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, cluster
 		query.Where("(name LIKE :keyword)").
 			Param("keyword", "%"+keyword+"%")
 	}
+	if len(storageType) > 0 {
+		query.Attr("type", storageType)
+	}
 	ones, err := query.
 		ResultPk().
 		Offset(offset).

internal/db/models/http_cache_policy_model.go

@@ -2,39 +2,41 @@ package models
 
 // HTTPCachePolicy HTTP缓存策略
 type HTTPCachePolicy struct {
-	Id          uint32 `field:"id"`          // ID
-	AdminId     uint32 `field:"adminId"`     // 管理员ID
-	UserId      uint32 `field:"userId"`      // 用户ID
-	TemplateId  uint32 `field:"templateId"`  // 模版ID
-	IsOn        uint8  `field:"isOn"`        // 是否启用
-	Name        string `field:"name"`        // 名称
-	Capacity    string `field:"capacity"`    // 容量数据
-	MaxKeys     uint64 `field:"maxKeys"`     // 最多Key值
-	MaxSize     string `field:"maxSize"`     // 最大缓存内容尺寸
-	Type        string `field:"type"`        // 存储类型
-	Options     string `field:"options"`     // 存储选项
-	CreatedAt   uint64 `field:"createdAt"`   // 创建时间
-	State       uint8  `field:"state"`       // 状态
-	Description string `field:"description"` // 描述
-	Refs        string `field:"refs"`        // 默认的缓存设置
+	Id                   uint32 `field:"id"`                   // ID
+	AdminId              uint32 `field:"adminId"`              // 管理员ID
+	UserId               uint32 `field:"userId"`               // 用户ID
+	TemplateId           uint32 `field:"templateId"`           // 模版ID
+	IsOn                 uint8  `field:"isOn"`                 // 是否启用
+	Name                 string `field:"name"`                 // 名称
+	Capacity             string `field:"capacity"`             // 容量数据
+	MaxKeys              uint64 `field:"maxKeys"`              // 最多Key值
+	MaxSize              string `field:"maxSize"`              // 最大缓存内容尺寸
+	Type                 string `field:"type"`                 // 存储类型
+	Options              string `field:"options"`              // 存储选项
+	CreatedAt            uint64 `field:"createdAt"`            // 创建时间
+	State                uint8  `field:"state"`                // 状态
+	Description          string `field:"description"`          // 描述
+	Refs                 string `field:"refs"`                 // 默认的缓存设置
+	SyncCompressionCache uint8  `field:"syncCompressionCache"` // 是否同步写入压缩缓存
 }
 
 type HTTPCachePolicyOperator struct {
-	Id          interface{} // ID
-	AdminId     interface{} // 管理员ID
-	UserId      interface{} // 用户ID
-	TemplateId  interface{} // 模版ID
-	IsOn        interface{} // 是否启用
-	Name        interface{} // 名称
-	Capacity    interface{} // 容量数据
-	MaxKeys     interface{} // 最多Key值
-	MaxSize     interface{} // 最大缓存内容尺寸
-	Type        interface{} // 存储类型
-	Options     interface{} // 存储选项
-	CreatedAt   interface{} // 创建时间
-	State       interface{} // 状态
-	Description interface{} // 描述
-	Refs        interface{} // 默认的缓存设置
+	Id                   interface{} // ID
+	AdminId              interface{} // 管理员ID
+	UserId               interface{} // 用户ID
+	TemplateId           interface{} // 模版ID
+	IsOn                 interface{} // 是否启用
+	Name                 interface{} // 名称
+	Capacity             interface{} // 容量数据
+	MaxKeys              interface{} // 最多Key值
+	MaxSize              interface{} // 最大缓存内容尺寸
+	Type                 interface{} // 存储类型
+	Options              interface{} // 存储选项
+	CreatedAt            interface{} // 创建时间
+	State                interface{} // 状态
+	Description          interface{} // 描述
+	Refs                 interface{} // 默认的缓存设置
+	SyncCompressionCache interface{} // 是否同步写入压缩缓存
 }
 
 func NewHTTPCachePolicyOperator() *HTTPCachePolicyOperator {

internal/db/models/ip_list_dao.go

@@ -184,6 +184,10 @@ func (this *IPListDAO) IncreaseVersion(tx *dbs.Tx) (int64, error) {
 
 // CheckUserIPList 检查用户权限
 func (this *IPListDAO) CheckUserIPList(tx *dbs.Tx, userId int64, listId int64) error {
+	if userId == 0 || listId == 0 {
+		return ErrNotFound
+	}
+
 	ok, err := this.Query(tx).
 		Pk(listId).
 		Attr("userId", userId).
@@ -194,6 +198,18 @@ func (this *IPListDAO) CheckUserIPList(tx *dbs.Tx, userId int64, listId int64) e
 	if ok {
 		return nil
 	}
+
+	// 检查是否被用户的服务所使用
+	policyIds, err := SharedHTTPFirewallPolicyDAO.FindEnabledFirewallPolicyIdsWithIPListId(tx, listId)
+	if err != nil {
+		return err
+	}
+	for _, policyId := range policyIds {
+		if SharedHTTPFirewallPolicyDAO.CheckUserFirewallPolicy(tx, userId, policyId) == nil {
+			return nil
+		}
+	}
+
 	return ErrNotFound
 }
 

internal/db/models/log_dao.go

@@ -33,7 +33,7 @@ func init() {
 	})
 }
 
-// 创建管理员日志
+// CreateLog 创建管理员日志
 func (this *LogDAO) CreateLog(tx *dbs.Tx, adminType string, adminId int64, level string, description string, action string, ip string) error {
 	op := NewLogOperator()
 	op.Level = level
@@ -57,7 +57,7 @@ func (this *LogDAO) CreateLog(tx *dbs.Tx, adminType string, adminId int64, level
 	return err
 }
 
-// 计算所有日志数量
+// CountLogs 计算所有日志数量
 func (this *LogDAO) CountLogs(tx *dbs.Tx, dayFrom string, dayTo string, keyword string, userType string) (int64, error) {
 	dayFrom = this.formatDay(dayFrom)
 	dayTo = this.formatDay(dayTo)
@@ -86,7 +86,7 @@ func (this *LogDAO) CountLogs(tx *dbs.Tx, dayFrom string, dayTo string, keyword
 	return query.Count()
 }
 
-// 列出单页日志
+// ListLogs 列出单页日志
 func (this *LogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64, dayFrom string, dayTo string, keyword string, userType string) (result []*Log, err error) {
 	dayFrom = this.formatDay(dayFrom)
 	dayTo = this.formatDay(dayTo)
@@ -120,7 +120,7 @@ func (this *LogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64, dayFrom strin
 	return
 }
 
-// 物理删除日志
+// DeleteLogPermanently 物理删除日志
 func (this *LogDAO) DeleteLogPermanently(tx *dbs.Tx, logId int64) error {
 	if logId <= 0 {
 		return errors.New("invalid logId")
@@ -129,14 +129,14 @@ func (this *LogDAO) DeleteLogPermanently(tx *dbs.Tx, logId int64) error {
 	return err
 }
 
-// 物理删除所有日志
+// DeleteAllLogsPermanently 物理删除所有日志
 func (this *LogDAO) DeleteAllLogsPermanently(tx *dbs.Tx) error {
 	_, err := this.Query(tx).
 		Delete()
 	return err
 }
 
-// 物理删除某些天之前的日志
+// DeleteLogsPermanentlyBeforeDays 物理删除某些天之前的日志
 func (this *LogDAO) DeleteLogsPermanentlyBeforeDays(tx *dbs.Tx, days int) error {
 	if days <= 0 {
 		days = 0
@@ -148,7 +148,7 @@ func (this *LogDAO) DeleteLogsPermanentlyBeforeDays(tx *dbs.Tx, days int) error
 	return err
 }
 
-// 计算当前日志容量大小
+// SumLogsSize 计算当前日志容量大小
 func (this *LogDAO) SumLogsSize() (int64, error) {
 	col, err := this.Instance.FindCol(0, "SELECT DATA_LENGTH FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=? LIMIT 1", this.Instance.Name(), this.Table)
 	if err != nil {

internal/db/models/node_cluster_dao.go

@@ -207,6 +207,14 @@ func (this *NodeClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name stri
 	return this.NotifyUpdate(tx, clusterId)
 }
 
+// UpdateClusterIsPinned 设置集群是否置顶
+func (this *NodeClusterDAO) UpdateClusterIsPinned(tx *dbs.Tx, clusterId int64, isPinned bool) error {
+	return this.Query(tx).
+		Pk(clusterId).
+		Set("isPinned", isPinned).
+		UpdateQuickly()
+}
+
 // CountAllEnabledClusters 计算所有集群数量
 func (this *NodeClusterDAO) CountAllEnabledClusters(tx *dbs.Tx, keyword string) (int64, error) {
 	query := this.Query(tx).
@@ -230,6 +238,7 @@ func (this *NodeClusterDAO) ListEnabledClusters(tx *dbs.Tx, keyword string, offs
 		Offset(offset).
 		Limit(size).
 		Slice(&result).
+		Desc("isPinned").
 		DescPk().
 		FindAll()
 	return

internal/db/models/node_cluster_model.go

@@ -30,6 +30,7 @@ type NodeCluster struct {
 	NodeMaxThreads        uint32 `field:"nodeMaxThreads"`        // 节点最大线程数
 	NodeTCPMaxConnections uint32 `field:"nodeTCPMaxConnections"` // TCP最大连接数
 	AutoOpenPorts         uint8  `field:"autoOpenPorts"`         // 是否自动尝试开放端口
+	IsPinned              uint8  `field:"isPinned"`              // 是否置顶
 }
 
 type NodeClusterOperator struct {
@@ -61,6 +62,7 @@ type NodeClusterOperator struct {
 	NodeMaxThreads        interface{} // 节点最大线程数
 	NodeTCPMaxConnections interface{} // TCP最大连接数
 	AutoOpenPorts         interface{} // 是否自动尝试开放端口
+	IsPinned              interface{} // 是否置顶
 }
 
 func NewNodeClusterOperator() *NodeClusterOperator {

internal/db/models/node_dao.go

@@ -456,9 +456,26 @@ func (this *NodeDAO) FindEnabledNodeClusterIds(tx *dbs.Tx, nodeId int64) (result
 	return
 }
 
+// FindEnabledNodeIdsWithClusterId 查找某个集群下的所有节点IDs
+func (this *NodeDAO) FindEnabledNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64) ([]int64, error) {
+	ones, err := this.Query(tx).
+		Attr("clusterId", clusterId).
+		State(NodeClusterStateEnabled).
+		ResultPk().
+		FindAll()
+	if err != nil {
+		return nil, err
+	}
+	var result = []int64{}
+	for _, one := range ones {
+		result = append(result, int64(one.(*Node).Id))
+	}
+	return result, nil
+}
+
 // FindAllNodeIdsMatch 匹配节点并返回节点ID
 func (this *NodeDAO) FindAllNodeIdsMatch(tx *dbs.Tx, clusterId int64, includeSecondaryNodes bool, isOn configutils.BoolState) (result []int64, err error) {
-	query := this.Query(tx)
+	var query = this.Query(tx)
 	query.State(NodeStateEnabled)
 	if clusterId > 0 {
 		if includeSecondaryNodes {
@@ -856,6 +873,8 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 		}
 	}
 
+	config.CacheDiskDir = node.CacheDiskDir
+
 	// TOA
 	toaConfig, err := SharedNodeClusterDAO.FindClusterTOAConfig(tx, primaryClusterId, cacheMap)
 	if err != nil {
@@ -929,6 +948,13 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 		}
 	}
 
+	// OCSP
+	ocspVersion, err := SharedSSLCertDAO.FindCertOCSPLatestVersion(tx)
+	if err != nil {
+		return nil, err
+	}
+	config.OCSPVersion = ocspVersion
+
 	return config, nil
 }
 
@@ -1205,7 +1231,7 @@ func (this *NodeDAO) UpdateNodeSystem(tx *dbs.Tx, nodeId int64, maxCPU int32) er
 }
 
 // UpdateNodeCache 设置缓存相关
-func (this *NodeDAO) UpdateNodeCache(tx *dbs.Tx, nodeId int64, maxCacheDiskCapacityJSON []byte, maxCacheMemoryCapacityJSON []byte) error {
+func (this *NodeDAO) UpdateNodeCache(tx *dbs.Tx, nodeId int64, maxCacheDiskCapacityJSON []byte, maxCacheMemoryCapacityJSON []byte, cacheDiskDir string) error {
 	if nodeId <= 0 {
 		return errors.New("invalid nodeId")
 	}
@@ -1217,6 +1243,7 @@ func (this *NodeDAO) UpdateNodeCache(tx *dbs.Tx, nodeId int64, maxCacheDiskCapac
 	if len(maxCacheMemoryCapacityJSON) > 0 {
 		op.MaxCacheMemoryCapacity = maxCacheMemoryCapacityJSON
 	}
+	op.CacheDiskDir = cacheDiskDir
 	err := this.Save(tx, op)
 	if err != nil {
 		return err

internal/db/models/node_model.go

@@ -31,6 +31,7 @@ type Node struct {
 	DnsRoutes              string `field:"dnsRoutes"`              // DNS线路设置
 	MaxCacheDiskCapacity   string `field:"maxCacheDiskCapacity"`   // 硬盘缓存容量
 	MaxCacheMemoryCapacity string `field:"maxCacheMemoryCapacity"` // 内存缓存容量
+	CacheDiskDir           string `field:"cacheDiskDir"`           // 缓存目录
 }
 
 type NodeOperator struct {
@@ -63,6 +64,7 @@ type NodeOperator struct {
 	DnsRoutes              interface{} // DNS线路设置
 	MaxCacheDiskCapacity   interface{} // 硬盘缓存容量
 	MaxCacheMemoryCapacity interface{} // 内存缓存容量
+	CacheDiskDir           interface{} // 缓存目录
 }
 
 func NewNodeOperator() *NodeOperator {

internal/db/models/node_task_dao.go

@@ -53,8 +53,8 @@ func (this *NodeTaskDAO) CreateNodeTask(tx *dbs.Tx, role string, clusterId int64
 	if clusterId <= 0 || nodeId <= 0 {
 		return nil
 	}
-	uniqueId := role + "@" + types.String(nodeId) + "@node@" + taskType
-	updatedAt := time.Now().Unix()
+	var uniqueId = role + "@" + types.String(nodeId) + "@node@" + types.String(serverId) + "@" + taskType
+	var updatedAt = time.Now().Unix()
 	_, _, err := this.Query(tx).
 		InsertOrUpdate(maps.Map{
 			"role":      role,
@@ -76,6 +76,7 @@ func (this *NodeTaskDAO) CreateNodeTask(tx *dbs.Tx, role string, clusterId int64
 			"error":      "",
 			"isNotified": 0,
 			"version":    version,
+			"serverId":   serverId,
 		})
 	return err
 }
@@ -86,8 +87,8 @@ func (this *NodeTaskDAO) CreateClusterTask(tx *dbs.Tx, role string, clusterId in
 		return nil
 	}
 
-	uniqueId := role + "@" + types.String(clusterId) + "@" + types.String(serverId) + "@cluster@" + taskType
-	updatedAt := time.Now().Unix()
+	var uniqueId = role + "@" + types.String(clusterId) + "@" + types.String(serverId) + "@cluster@" + types.String(serverId) + "@" + taskType
+	var updatedAt = time.Now().Unix()
 	_, _, err := this.Query(tx).
 		InsertOrUpdate(maps.Map{
 			"role":       role,
@@ -109,6 +110,7 @@ func (this *NodeTaskDAO) CreateClusterTask(tx *dbs.Tx, role string, clusterId in
 			"isNotified": 0,
 			"error":      "",
 			"version":    time.Now().UnixNano(),
+			"serverId":   serverId,
 		})
 	return err
 }

internal/db/models/origin_dao.go

@@ -100,8 +100,9 @@ func (this *OriginDAO) CreateOrigin(tx *dbs.Tx,
 	maxConns int32,
 	maxIdleConns int32,
 	certRef *sslconfigs.SSLCertRef,
-	domains []string) (originId int64, err error) {
-	op := NewOriginOperator()
+	domains []string,
+	host string) (originId int64, err error) {
+	var op = NewOriginOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.IsOn = isOn
@@ -165,6 +166,8 @@ func (this *OriginDAO) CreateOrigin(tx *dbs.Tx,
 		op.Domains = "[]"
 	}
 
+	op.Host = host
+
 	op.State = OriginStateEnabled
 	err = this.Save(tx, op)
 	if err != nil {
@@ -187,11 +190,12 @@ func (this *OriginDAO) UpdateOrigin(tx *dbs.Tx,
 	maxConns int32,
 	maxIdleConns int32,
 	certRef *sslconfigs.SSLCertRef,
-	domains []string) error {
+	domains []string,
+	host string) error {
 	if originId <= 0 {
 		return errors.New("invalid originId")
 	}
-	op := NewOriginOperator()
+	var op = NewOriginOperator()
 	op.Id = originId
 	op.Name = name
 	op.Addr = addrJSON
@@ -257,6 +261,8 @@ func (this *OriginDAO) UpdateOrigin(tx *dbs.Tx,
 		op.Domains = "[]"
 	}
 
+	op.Host = host
+
 	err := this.Save(tx, op)
 	if err != nil {
 		return err
@@ -284,7 +290,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 		return nil, nil
 	}
 
-	config := &serverconfigs.OriginConfig{
+	var config = &serverconfigs.OriginConfig{
 		Id:           int64(origin.Id),
 		IsOn:         origin.IsOn == 1,
 		Version:      int(origin.Version),

internal/db/models/reverse_proxy_dao.go

@@ -107,6 +107,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 	config.RequestURI = reverseProxy.RequestURI
 	config.StripPrefix = reverseProxy.StripPrefix
 	config.AutoFlush = reverseProxy.AutoFlush == 1
+	config.FollowRedirects = reverseProxy.FollowRedirects == 1
 
 	schedulingConfig := &serverconfigs.SchedulingConfig{}
 	if len(reverseProxy.Scheduling) > 0 && reverseProxy.Scheduling != "null" {
@@ -312,12 +313,13 @@ func (this *ReverseProxyDAO) UpdateReverseProxy(tx *dbs.Tx,
 	idleTimeout *shared.TimeDuration,
 	maxConns int32,
 	maxIdleConns int32,
-	proxyProtocolJSON []byte) error {
+	proxyProtocolJSON []byte,
+	followRedirects bool) error {
 	if reverseProxyId <= 0 {
 		return errors.New("invalid reverseProxyId")
 	}
 
-	op := NewReverseProxyOperator()
+	var op = NewReverseProxyOperator()
 	op.Id = reverseProxyId
 
 	if requestHostType < 0 {
@@ -329,6 +331,7 @@ func (this *ReverseProxyDAO) UpdateReverseProxy(tx *dbs.Tx,
 	op.RequestURI = requestURI
 	op.StripPrefix = stripPrefix
 	op.AutoFlush = autoFlush
+	op.FollowRedirects = followRedirects
 
 	if len(addHeaders) == 0 {
 		addHeaders = []string{}

internal/db/models/reverse_proxy_model.go

@@ -24,6 +24,7 @@ type ReverseProxy struct {
 	MaxConns        uint32 `field:"maxConns"`        // 最大并发连接数
 	MaxIdleConns    uint32 `field:"maxIdleConns"`    // 最大空闲连接数
 	ProxyProtocol   string `field:"proxyProtocol"`   // Proxy Protocol配置
+	FollowRedirects uint8  `field:"followRedirects"` // 回源跟随
 }
 
 type ReverseProxyOperator struct {
@@ -49,6 +50,7 @@ type ReverseProxyOperator struct {
 	MaxConns        interface{} // 最大并发连接数
 	MaxIdleConns    interface{} // 最大空闲连接数
 	ProxyProtocol   interface{} // Proxy Protocol配置
+	FollowRedirects interface{} // 回源跟随
 }
 
 func NewReverseProxyOperator() *ReverseProxyOperator {

internal/db/models/server_dao.go

@@ -1227,6 +1227,16 @@ func (this *ServerDAO) FindAllEnabledServerIdsWithSSLPolicyIds(tx *dbs.Tx, sslPo
 	return
 }
 
+// ExistEnabledUserServerWithSSLPolicyId 检查是否存在某个用户的策略
+func (this *ServerDAO) ExistEnabledUserServerWithSSLPolicyId(tx *dbs.Tx, userId int64, sslPolicyId int64) (bool, error) {
+	return this.Query(tx).
+		State(ServerStateEnabled).
+		Attr("userId", userId).
+		Where("(JSON_CONTAINS(https, :jsonQuery) OR JSON_CONTAINS(tls, :jsonQuery))").
+		Param("jsonQuery", maps.Map{"sslPolicyRef": maps.Map{"sslPolicyId": sslPolicyId}}.AsJSON()).
+		Exist()
+}
+
 // CountEnabledServersWithWebIds 计算使用某个缓存策略的所有服务数量
 func (this *ServerDAO) CountEnabledServersWithWebIds(tx *dbs.Tx, webIds []int64) (count int64, err error) {
 	if len(webIds) == 0 {

internal/db/models/ssl_cert_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
@@ -113,6 +114,8 @@ func (this *SSLCertDAO) CreateCert(tx *dbs.Tx, adminId int64, userId int64, isOn
 	}
 	op.CommonNames = commonNamesJSON
 
+	op.OcspIsUpdated = false
+
 	err = this.Save(tx, op)
 	if err != nil {
 		return 0, err
@@ -121,11 +124,33 @@ func (this *SSLCertDAO) CreateCert(tx *dbs.Tx, adminId int64, userId int64, isOn
 }
 
 // UpdateCert 修改证书
-func (this *SSLCertDAO) UpdateCert(tx *dbs.Tx, certId int64, isOn bool, name string, description string, serverName string, isCA bool, certData []byte, keyData []byte, timeBeginAt int64, timeEndAt int64, dnsNames []string, commonNames []string) error {
+func (this *SSLCertDAO) UpdateCert(tx *dbs.Tx,
+	certId int64,
+	isOn bool,
+	name string,
+	description string,
+	serverName string,
+	isCA bool,
+	certData []byte,
+	keyData []byte,
+	timeBeginAt int64,
+	timeEndAt int64,
+	dnsNames []string, commonNames []string) error {
 	if certId <= 0 {
 		return errors.New("invalid certId")
 	}
-	op := NewSSLCertOperator()
+
+	oldOne, err := this.Query(tx).Find()
+	if err != nil {
+		return err
+	}
+	if oldOne == nil {
+		return nil
+	}
+	var oldCert = oldOne.(*SSLCert)
+	var dataIsChanged = bytes.Compare(certData, []byte(oldCert.CertData)) != 0 || bytes.Compare(keyData, []byte(oldCert.KeyData)) != 0
+
+	var op = NewSSLCertOperator()
 	op.Id = certId
 	op.IsOn = isOn
 	op.Name = name
@@ -156,6 +181,16 @@ func (this *SSLCertDAO) UpdateCert(tx *dbs.Tx, certId int64, isOn bool, name str
 	}
 	op.CommonNames = commonNamesJSON
 
+	// OCSP
+	if dataIsChanged {
+		op.OcspIsUpdated = 0
+		op.Ocsp = ""
+		op.OcspUpdatedAt = 0
+		op.OcspError = ""
+		op.OcspTries = 0
+		op.OcspExpiresAt = 0
+	}
+
 	err = this.Save(tx, op)
 	if err != nil {
 		return err
@@ -195,6 +230,13 @@ func (this *SSLCertDAO) ComposeCertConfig(tx *dbs.Tx, certId int64, cacheMap *ut
 	config.TimeBeginAt = int64(cert.TimeBeginAt)
 	config.TimeEndAt = int64(cert.TimeEndAt)
 
+	// OCSP
+	if int64(cert.OcspExpiresAt) > time.Now().Unix() {
+		config.OCSP = []byte(cert.Ocsp)
+		config.OCSPExpiresAt = int64(cert.OcspExpiresAt)
+	}
+	config.OCSPError = cert.OcspError
+
 	if IsNotNull(cert.DnsNames) {
 		dnsNames := []string{}
 		err := json.Unmarshal([]byte(cert.DnsNames), &dnsNames)
@@ -356,6 +398,197 @@ func (this *SSLCertDAO) CheckUserCert(tx *dbs.Tx, certId int64, userId int64) er
 	return nil
 }
 
+// ListCertsToUpdateOCSP 查找需要更新OCSP的证书
+func (this *SSLCertDAO) ListCertsToUpdateOCSP(tx *dbs.Tx, maxTries int, size int64) (result []*SSLCert, err error) {
+	var nowTime = time.Now().Unix()
+	var query = this.Query(tx).
+		State(SSLCertStateEnabled).
+		Lt("ocspExpiresAt", nowTime+120). // 提前 N 秒钟准备更新
+		Lt("ocspTries", maxTries).
+		Lt("timeBeginAt", nowTime).
+		Gt("timeEndAt", nowTime)
+
+	// TODO 需要排除没有被server使用的policy，或许可以增加一个字段记录policy最近使用时间
+
+	// 检查函数
+	var JSONArrayAggIsEnabled = false
+	_, err = this.Object().Instance.Exec("SELECT JSON_ARRAYAGG('1')")
+	if err == nil {
+		JSONArrayAggIsEnabled = true
+	}
+
+	if JSONArrayAggIsEnabled {
+		query.Where("JSON_CONTAINS((SELECT JSON_ARRAYAGG(JSON_EXTRACT(certs, '$[*].certId')) FROM edgeSSLPolicies WHERE state=1 AND ocspIsOn=1 AND certs IS NOT NULL), CAST(id AS CHAR))")
+	} else {
+		query.Where("JSON_CONTAINS((SELECT REPLACE(GROUP_CONCAT(JSON_EXTRACT(certs, '$[*].certId')), '],[', ',') FROM edgeSSLPolicies WHERE state=1 AND ocspIsOn=1 AND certs IS NOT NULL), CAST(id AS CHAR))")
+	}
+
+	_, err = query.
+		Asc("ocspUpdatedAt").
+		Limit(size).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// ListCertOCSPAfterVersion 列出某个版本后的OCSP
+func (this *SSLCertDAO) ListCertOCSPAfterVersion(tx *dbs.Tx, version int64, size int64) (result []*SSLCert, err error) {
+	// 不需要判断ocsp是否为空
+	_, err = this.Query(tx).
+		Result("id", "ocsp", "ocspUpdatedVersion", "ocspExpiresAt").
+		State(SSLCertStateEnabled).
+		Attr("ocspIsUpdated", 1).
+		Gt("ocspUpdatedVersion", version).
+		Asc("ocspUpdatedVersion").
+		Limit(size).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// FindCertOCSPLatestVersion 获取OCSP最新版本
+func (this *SSLCertDAO) FindCertOCSPLatestVersion(tx *dbs.Tx) (int64, error) {
+	return this.Query(tx).
+		Result("ocspUpdatedVersion").
+		Desc("ocspUpdatedVersion").
+		Limit(1).
+		FindInt64Col(0)
+}
+
+// PrepareCertOCSPUpdating 更新OCSP更新时间，以便于准备更新，相当于锁定
+func (this *SSLCertDAO) PrepareCertOCSPUpdating(tx *dbs.Tx, certId int64) error {
+	return this.Query(tx).
+		Pk(certId).
+		Set("ocspUpdatedAt", time.Now().Unix()).
+		UpdateQuickly()
+
+}
+
+// UpdateCertOCSP 修改OCSP
+func (this *SSLCertDAO) UpdateCertOCSP(tx *dbs.Tx, certId int64, ocsp []byte, expiresAt int64, hasErr bool, errString string) error {
+	if hasErr && len(errString) == 0 {
+		errString = "failed"
+	}
+
+	version, err := SharedSysLockerDAO.Increase(tx, "SSL_CERT_OCSP_VERSION", 1)
+	if err != nil {
+		return err
+	}
+
+	if ocsp == nil {
+		ocsp = []byte{}
+	}
+
+	// 限制长度
+	if len(errString) > 300 {
+		errString = errString[:300]
+	}
+
+	var query = this.Query(tx).
+		Pk(certId).
+		Set("ocsp", ocsp).
+		Set("ocspError", errString).
+		Set("ocspIsUpdated", true).
+		Set("ocspUpdatedAt", time.Now().Unix()).
+		Set("ocspUpdatedVersion", version).
+		Set("ocspExpiresAt", expiresAt)
+
+	if hasErr {
+		query.Set("ocspTries", dbs.SQL("ocspTries+1"))
+	} else {
+		query.Set("ocspTries", 0)
+	}
+
+	err = query.UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	// 注意：这里不通知更新，避免频繁的更新导致服务不稳定
+	return nil
+}
+
+// CountAllSSLCertsWithOCSPError 计算有OCSP错误的证书数量
+func (this *SSLCertDAO) CountAllSSLCertsWithOCSPError(tx *dbs.Tx, keyword string) (int64, error) {
+	var query = this.Query(tx)
+
+	if len(keyword) > 0 {
+		query.Where("(name LIKE :keyword OR description LIKE :keyword OR dnsNames LIKE :keyword OR commonNames LIKE :keyword OR ocspError LIKE :keyword)").
+			Param("keyword", "%"+keyword+"%")
+	}
+
+	return query.
+		State(SSLCertStateEnabled).
+		Attr("ocspIsUpdated", true).
+		Where("LENGTH(ocspError) > 0").
+		Count()
+}
+
+// ListSSLCertsWithOCSPError 列出有OCSP错误的证书
+func (this *SSLCertDAO) ListSSLCertsWithOCSPError(tx *dbs.Tx, keyword string, offset int64, size int64) (result []*SSLCert, err error) {
+	var query = this.Query(tx)
+
+	if len(keyword) > 0 {
+		query.Where("(name LIKE :keyword OR description LIKE :keyword OR dnsNames LIKE :keyword OR commonNames LIKE :keyword OR ocspError LIKE :keyword)").
+			Param("keyword", "%"+keyword+"%")
+	}
+
+	_, err = query.
+		State(SSLCertStateEnabled).
+		Attr("ocspIsUpdated", true).
+		Where("LENGTH(ocspError) > 0").
+		Offset(offset).
+		Limit(size).
+		DescPk().
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// IgnoreSSLCertsWithOCSPError 忽略一组OCSP证书错误
+func (this *SSLCertDAO) IgnoreSSLCertsWithOCSPError(tx *dbs.Tx, certIds []int64) error {
+	for _, certId := range certIds {
+		err := this.Query(tx).
+			Pk(certId).
+			Set("ocspError", "").
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ResetSSLCertsWithOCSPError 重置一组证书OCSP错误状态
+func (this *SSLCertDAO) ResetSSLCertsWithOCSPError(tx *dbs.Tx, certIds []int64) error {
+	for _, certId := range certIds {
+		err := this.Query(tx).
+			Pk(certId).
+			Set("ocspIsUpdated", 0).
+			Set("ocspUpdatedAt", 0).
+			Set("ocspError", "").
+			Set("ocspTries", 0).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ResetAllSSLCertsWithOCSPError 重置所有证书OCSP错误状态
+func (this *SSLCertDAO) ResetAllSSLCertsWithOCSPError(tx *dbs.Tx) error {
+	return this.Query(tx).
+		State(SSLCertStateEnabled).
+		Attr("ocspIsUpdated", 1).
+		Where("LENGTH(ocspError)>0").
+		Set("ocspIsUpdated", 0).
+		Set("ocspUpdatedAt", 0).
+		Set("ocspError", "").
+		Set("ocspTries", 0).
+		UpdateQuickly()
+}
+
 // NotifyUpdate 通知更新
 func (this *SSLCertDAO) NotifyUpdate(tx *dbs.Tx, certId int64) error {
 	policyIds, err := SharedSSLPolicyDAO.FindAllEnabledPolicyIdsWithCertId(tx, certId)

internal/db/models/ssl_cert_dao_test.go

@@ -1,5 +1,19 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"testing"
 )
+
+func TestSSLCertDAO_ListCertsToUpdateOCSP(t *testing.T) {
+	var dao = models.NewSSLCertDAO()
+	certs, err := dao.ListCertsToUpdateOCSP(nil, 3600, 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, cert := range certs {
+		t.Log(cert.Id, cert.Name, "updatedAt:", cert.OcspUpdatedAt, timeutil.FormatTime("Y-m-d H:i:s", int64(cert.OcspUpdatedAt)), cert.OcspIsUpdated)
+	}
+}

internal/db/models/ssl_cert_model.go

@@ -1,52 +1,66 @@
 package models
 
-// SSL证书
+// SSLCert SSL证书
 type SSLCert struct {
-	Id          uint32 `field:"id"`          // ID
-	AdminId     uint32 `field:"adminId"`     // 管理员ID
-	UserId      uint32 `field:"userId"`      // 用户ID
-	State       uint8  `field:"state"`       // 状态
-	CreatedAt   uint64 `field:"createdAt"`   // 创建时间
-	UpdatedAt   uint64 `field:"updatedAt"`   // 修改时间
-	IsOn        uint8  `field:"isOn"`        // 是否启用
-	Name        string `field:"name"`        // 证书名
-	Description string `field:"description"` // 描述
-	CertData    string `field:"certData"`    // 证书内容
-	KeyData     string `field:"keyData"`     // 密钥内容
-	ServerName  string `field:"serverName"`  // 证书使用的主机名
-	IsCA        uint8  `field:"isCA"`        // 是否为CA证书
-	GroupIds    string `field:"groupIds"`    // 证书分组
-	TimeBeginAt uint64 `field:"timeBeginAt"` // 开始时间
-	TimeEndAt   uint64 `field:"timeEndAt"`   // 结束时间
-	DnsNames    string `field:"dnsNames"`    // DNS名称列表
-	CommonNames string `field:"commonNames"` // 发行单位列表
-	IsACME      uint8  `field:"isACME"`      // 是否为ACME自动生成的
-	AcmeTaskId  uint64 `field:"acmeTaskId"`  // ACME任务ID
-	NotifiedAt  uint64 `field:"notifiedAt"`  // 最后通知时间
+	Id                 uint32 `field:"id"`                 // ID
+	AdminId            uint32 `field:"adminId"`            // 管理员ID
+	UserId             uint32 `field:"userId"`             // 用户ID
+	State              uint8  `field:"state"`              // 状态
+	CreatedAt          uint64 `field:"createdAt"`          // 创建时间
+	UpdatedAt          uint64 `field:"updatedAt"`          // 修改时间
+	IsOn               uint8  `field:"isOn"`               // 是否启用
+	Name               string `field:"name"`               // 证书名
+	Description        string `field:"description"`        // 描述
+	CertData           string `field:"certData"`           // 证书内容
+	KeyData            string `field:"keyData"`            // 密钥内容
+	ServerName         string `field:"serverName"`         // 证书使用的主机名
+	IsCA               uint8  `field:"isCA"`               // 是否为CA证书
+	GroupIds           string `field:"groupIds"`           // 证书分组
+	TimeBeginAt        uint64 `field:"timeBeginAt"`        // 开始时间
+	TimeEndAt          uint64 `field:"timeEndAt"`          // 结束时间
+	DnsNames           string `field:"dnsNames"`           // DNS名称列表
+	CommonNames        string `field:"commonNames"`        // 发行单位列表
+	IsACME             uint8  `field:"isACME"`             // 是否为ACME自动生成的
+	AcmeTaskId         uint64 `field:"acmeTaskId"`         // ACME任务ID
+	NotifiedAt         uint64 `field:"notifiedAt"`         // 最后通知时间
+	Ocsp               string `field:"ocsp"`               // OCSP缓存
+	OcspIsUpdated      uint8  `field:"ocspIsUpdated"`      // OCSP是否已更新
+	OcspUpdatedAt      uint64 `field:"ocspUpdatedAt"`      // OCSP更新时间
+	OcspError          string `field:"ocspError"`          // OCSP更新错误
+	OcspUpdatedVersion uint64 `field:"ocspUpdatedVersion"` // OCSP更新版本
+	OcspExpiresAt      uint64 `field:"ocspExpiresAt"`      // OCSP过期时间(UTC)
+	OcspTries          uint32 `field:"ocspTries"`          // OCSP尝试次数
 }
 
 type SSLCertOperator struct {
-	Id          interface{} // ID
-	AdminId     interface{} // 管理员ID
-	UserId      interface{} // 用户ID
-	State       interface{} // 状态
-	CreatedAt   interface{} // 创建时间
-	UpdatedAt   interface{} // 修改时间
-	IsOn        interface{} // 是否启用
-	Name        interface{} // 证书名
-	Description interface{} // 描述
-	CertData    interface{} // 证书内容
-	KeyData     interface{} // 密钥内容
-	ServerName  interface{} // 证书使用的主机名
-	IsCA        interface{} // 是否为CA证书
-	GroupIds    interface{} // 证书分组
-	TimeBeginAt interface{} // 开始时间
-	TimeEndAt   interface{} // 结束时间
-	DnsNames    interface{} // DNS名称列表
-	CommonNames interface{} // 发行单位列表
-	IsACME      interface{} // 是否为ACME自动生成的
-	AcmeTaskId  interface{} // ACME任务ID
-	NotifiedAt  interface{} // 最后通知时间
+	Id                 interface{} // ID
+	AdminId            interface{} // 管理员ID
+	UserId             interface{} // 用户ID
+	State              interface{} // 状态
+	CreatedAt          interface{} // 创建时间
+	UpdatedAt          interface{} // 修改时间
+	IsOn               interface{} // 是否启用
+	Name               interface{} // 证书名
+	Description        interface{} // 描述
+	CertData           interface{} // 证书内容
+	KeyData            interface{} // 密钥内容
+	ServerName         interface{} // 证书使用的主机名
+	IsCA               interface{} // 是否为CA证书
+	GroupIds           interface{} // 证书分组
+	TimeBeginAt        interface{} // 开始时间
+	TimeEndAt          interface{} // 结束时间
+	DnsNames           interface{} // DNS名称列表
+	CommonNames        interface{} // 发行单位列表
+	IsACME             interface{} // 是否为ACME自动生成的
+	AcmeTaskId         interface{} // ACME任务ID
+	NotifiedAt         interface{} // 最后通知时间
+	Ocsp               interface{} // OCSP缓存
+	OcspIsUpdated      interface{} // OCSP是否已更新
+	OcspUpdatedAt      interface{} // OCSP更新时间
+	OcspError          interface{} // OCSP更新错误
+	OcspUpdatedVersion interface{} // OCSP更新版本
+	OcspExpiresAt      interface{} // OCSP过期时间(UTC)
+	OcspTries          interface{} // OCSP尝试次数
 }
 
 func NewSSLCertOperator() *SSLCertOperator {

internal/db/models/ssl_cert_model_ext.go

@@ -1 +1,31 @@
 package models
+
+import "encoding/json"
+
+func (this *SSLCert) DecodeDNSNames() []string {
+	if len(this.DnsNames) == 0 {
+		return nil
+	}
+
+	var result = []string{}
+	var err = json.Unmarshal([]byte(this.DnsNames), &result)
+	if err != nil {
+		return nil
+	}
+
+	return result
+}
+
+func (this *SSLCert) DecodeCommonNames() []string {
+	if len(this.CommonNames) == 0 {
+		return nil
+	}
+
+	var result = []string{}
+	var err = json.Unmarshal([]byte(this.CommonNames), &result)
+	if err != nil {
+		return nil
+	}
+
+	return result
+}

internal/db/models/ssl_policy_dao.go

@@ -167,6 +167,9 @@ func (this *SSLPolicyDAO) ComposePolicyConfig(tx *dbs.Tx, policyId int64, cacheM
 		config.HSTS = hstsConfig
 	}
 
+	// ocsp
+	config.OCSPIsOn = policy.OcspIsOn == 1
+
 	if cacheMap != nil {
 		cacheMap.Put(cacheKey, config)
 	}
@@ -196,7 +199,7 @@ func (this *SSLPolicyDAO) FindAllEnabledPolicyIdsWithCertId(tx *dbs.Tx, certId i
 }
 
 // CreatePolicy 创建Policy
-func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) (int64, error) {
+func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, ocspIsOn bool, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) (int64, error) {
 	op := NewSSLPolicyOperator()
 	op.State = SSLPolicyStateEnabled
 	op.IsOn = true
@@ -213,6 +216,8 @@ func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64,
 		op.Hsts = hstsJSON
 	}
 
+	op.OcspIsOn = ocspIsOn
+
 	op.ClientAuthType = clientAuthType
 	if len(clientCACertsJSON) > 0 {
 		op.ClientCACerts = clientCACertsJSON
@@ -234,7 +239,7 @@ func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64,
 }
 
 // UpdatePolicy 修改Policy
-func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) error {
+func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, ocspIsOn bool, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) error {
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
@@ -251,6 +256,8 @@ func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled
 		op.Hsts = hstsJSON
 	}
 
+	op.OcspIsOn = ocspIsOn
+
 	op.ClientAuthType = clientAuthType
 	if len(clientCACertsJSON) > 0 {
 		op.ClientCACerts = clientCACertsJSON
@@ -274,9 +281,9 @@ func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled
 }
 
 // CheckUserPolicy 检查是否为用户所属策略
-func (this *SSLPolicyDAO) CheckUserPolicy(tx *dbs.Tx, policyId int64, userId int64) error {
+func (this *SSLPolicyDAO) CheckUserPolicy(tx *dbs.Tx, userId int64, policyId int64) error {
 	if policyId <= 0 || userId <= 0 {
-		return errors.New("not found")
+		return ErrNotFound
 	}
 	ok, err := this.Query(tx).
 		State(SSLPolicyStateEnabled).
@@ -287,7 +294,14 @@ func (this *SSLPolicyDAO) CheckUserPolicy(tx *dbs.Tx, policyId int64, userId int
 		return err
 	}
 	if !ok {
-		return errors.New("not found")
+		// 是否为当前用户的某个服务所用
+		exists, err := SharedServerDAO.ExistEnabledUserServerWithSSLPolicyId(tx, userId, policyId)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			return ErrNotFound
+		}
 	}
 	return nil
 }

internal/db/models/ssl_policy_model.go

@@ -1,6 +1,6 @@
 package models
 
-//
+// SSLPolicy SSL配置策略
 type SSLPolicy struct {
 	Id               uint32 `field:"id"`               // ID
 	AdminId          uint32 `field:"adminId"`          // 管理员ID
@@ -14,6 +14,7 @@ type SSLPolicy struct {
 	CipherSuites     string `field:"cipherSuites"`     // 加密算法套件
 	Hsts             string `field:"hsts"`             // HSTS设置
 	Http2Enabled     uint8  `field:"http2Enabled"`     // 是否启用HTTP/2
+	OcspIsOn         uint8  `field:"ocspIsOn"`         // 是否启用OCSP
 	State            uint8  `field:"state"`            // 状态
 	CreatedAt        uint64 `field:"createdAt"`        // 创建时间
 }
@@ -31,6 +32,7 @@ type SSLPolicyOperator struct {
 	CipherSuites     interface{} // 加密算法套件
 	Hsts             interface{} // HSTS设置
 	Http2Enabled     interface{} // 是否启用HTTP/2
+	OcspIsOn         interface{} // 是否启用OCSP
 	State            interface{} // 状态
 	CreatedAt        interface{} // 创建时间
 }

internal/db/models/stats/server_http_firewall_daily_stat_dao.go

@@ -109,10 +109,13 @@ func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId i
 }
 
 // FindDailyStats 查询某个日期段内的记录
-func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) {
+func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, actions []string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) {
+	if len(actions) == 0 {
+		return nil, nil
+	}
 	query := this.Query(tx).
 		Between("day", dayFrom, dayTo).
-		Attr("action", action)
+		Attr("action", actions)
 	if serverId > 0 {
 		query.Attr("serverId", serverId)
 	} else if userId > 0 {

internal/dnsclients/provider_alidns.go

@@ -150,13 +150,13 @@ func (this *AliDNSProvider) AddRecord(domain string, newRecord *dnstypes.Record)
 	resp := alidns.CreateAddDomainRecordResponse()
 	err := this.doAPI(req, resp)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 	if resp.IsSuccess() {
 		return nil
 	}
 
-	return errors.New(resp.GetHttpContentString())
+	return this.WrapError(errors.New(resp.GetHttpContentString()), domain, newRecord)
 }
 
 // UpdateRecord 修改记录
@@ -174,7 +174,7 @@ func (this *AliDNSProvider) UpdateRecord(domain string, record *dnstypes.Record,
 
 	resp := alidns.CreateUpdateDomainRecordResponse()
 	err := this.doAPI(req, resp)
-	return err
+	return this.WrapError(err, domain, newRecord)
 }
 
 // DeleteRecord 删除记录
@@ -184,7 +184,7 @@ func (this *AliDNSProvider) DeleteRecord(domain string, record *dnstypes.Record)
 
 	resp := alidns.CreateDeleteDomainRecordResponse()
 	err := this.doAPI(req, resp)
-	return err
+	return this.WrapError(err, domain, record)
 }
 
 // DefaultRoute 默认线路

internal/dnsclients/provider_base.go

@@ -1,4 +1,29 @@
 package dnsclients
 
+import (
+	"errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/dnsclients/dnstypes"
+	"github.com/iwind/TeaGo/types"
+)
+
 type BaseProvider struct {
 }
+
+// WrapError 封装解析相关错误
+func (this *BaseProvider) WrapError(err error, domain string, record *dnstypes.Record) error {
+	if err == nil {
+		return nil
+	}
+
+	if record == nil {
+		return err
+	}
+
+	var fullname = ""
+	if len(record.Name) == 0 {
+		fullname = domain
+	} else {
+		fullname = record.Name + "." + domain
+	}
+	return errors.New("record operation failed: '" + fullname + " " + record.Type + " " + record.Value + " " + types.String(record.TTL) + "': " + err.Error())
+}

internal/dnsclients/provider_base_test.go

@@ -0,0 +1,38 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package dnsclients_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/dnsclients"
+	"github.com/TeaOSLab/EdgeAPI/internal/dnsclients/dnstypes"
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"testing"
+)
+
+func TestBaseProvider_WrapError(t *testing.T) {
+	var provider = &dnsclients.BaseProvider{}
+	t.Log(provider.WrapError(nil, "example.com", &dnstypes.Record{
+		Id:    "",
+		Name:  "a",
+		Type:  "A",
+		Value: "192.168.1.100",
+		Route: "",
+		TTL:   3600,
+	}))
+	t.Log(provider.WrapError(errors.New("fake error"), "example.com", &dnstypes.Record{
+		Id:    "",
+		Name:  "a",
+		Type:  "A",
+		Value: "192.168.1.100",
+		Route: "",
+		TTL:   3600,
+	}))
+	t.Log(provider.WrapError(errors.New("fake error"), "example.com", &dnstypes.Record{
+		Id:    "",
+		Name:  "",
+		Type:  "A",
+		Value: "192.168.1.100",
+		Route: "",
+		TTL:   3600,
+	}))
+}

internal/dnsclients/provider_cloud_flare.go

@@ -169,7 +169,7 @@ func (this *CloudFlareProvider) QueryRecord(domain string, name string, recordTy
 func (this *CloudFlareProvider) AddRecord(domain string, newRecord *dnstypes.Record) error {
 	zoneId, err := this.findZoneIdWithDomain(domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	resp := new(cloudflare.CreateDNSRecordResponse)
@@ -186,7 +186,7 @@ func (this *CloudFlareProvider) AddRecord(domain string, newRecord *dnstypes.Rec
 		"ttl":     ttl,
 	}, resp)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 	return nil
 }
@@ -195,7 +195,7 @@ func (this *CloudFlareProvider) AddRecord(domain string, newRecord *dnstypes.Rec
 func (this *CloudFlareProvider) UpdateRecord(domain string, record *dnstypes.Record, newRecord *dnstypes.Record) error {
 	zoneId, err := this.findZoneIdWithDomain(domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	var ttl = newRecord.TTL
@@ -216,13 +216,13 @@ func (this *CloudFlareProvider) UpdateRecord(domain string, record *dnstypes.Rec
 func (this *CloudFlareProvider) DeleteRecord(domain string, record *dnstypes.Record) error {
 	zoneId, err := this.findZoneIdWithDomain(domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, record)
 	}
 
 	resp := new(cloudflare.DeleteDNSRecordResponse)
 	err = this.doAPI(http.MethodDelete, "zones/"+zoneId+"/dns_records/"+record.Id, map[string]string{}, nil, resp)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, record)
 	}
 	return nil
 }

internal/dnsclients/provider_custom_http.go

@@ -29,6 +29,8 @@ var customHTTPClient = &http.Client{
 type CustomHTTPProvider struct {
 	url    string
 	secret string
+
+	BaseProvider
 }
 
 // Auth 认证
@@ -96,7 +98,7 @@ func (this *CustomHTTPProvider) QueryRecord(domain string, name string, recordTy
 	if err != nil {
 		return nil, err
 	}
-	if len(resp) == 0 {
+	if len(resp) == 0 || string(resp) == "null" {
 		return nil, nil
 	}
 	record := &dnstypes.Record{}
@@ -117,7 +119,7 @@ func (this *CustomHTTPProvider) AddRecord(domain string, newRecord *dnstypes.Rec
 		"domain":    domain,
 		"newRecord": newRecord,
 	})
-	return err
+	return this.WrapError(err, domain, newRecord)
 }
 
 // UpdateRecord 修改记录
@@ -128,7 +130,7 @@ func (this *CustomHTTPProvider) UpdateRecord(domain string, record *dnstypes.Rec
 		"record":    record,
 		"newRecord": newRecord,
 	})
-	return err
+	return this.WrapError(err, domain, newRecord)
 }
 
 // DeleteRecord 删除记录
@@ -138,7 +140,7 @@ func (this *CustomHTTPProvider) DeleteRecord(domain string, record *dnstypes.Rec
 		"domain": domain,
 		"record": record,
 	})
-	return err
+	return this.WrapError(err, domain, record)
 }
 
 // DefaultRoute 默认线路
@@ -162,10 +164,11 @@ func (this *CustomHTTPProvider) post(params maps.Map) (respData []byte, err erro
 	if err != nil {
 		return nil, err
 	}
-	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+	var timestamp = strconv.FormatInt(time.Now().Unix(), 10)
 	req.Header.Set("Timestamp", timestamp)
 	req.Header.Set("Token", fmt.Sprintf("%x", sha1.Sum([]byte(this.secret+"@"+timestamp))))
 	req.Header.Set("User-Agent", teaconst.ProductName+"/"+teaconst.Version)
+	req.Header.Set("Content-Type", "application/json")
 
 	resp, err := customHTTPClient.Do(req)
 	if err != nil {

internal/dnsclients/provider_dnspod.go

@@ -182,7 +182,7 @@ func (this *DNSPodProvider) AddRecord(domain string, newRecord *dnstypes.Record)
 		args["ttl"] = types.String(newRecord.TTL)
 	}
 	_, err := this.post("/Record.Create", args)
-	return err
+	return this.WrapError(err, domain, newRecord)
 }
 
 // UpdateRecord 修改记录
@@ -211,7 +211,7 @@ func (this *DNSPodProvider) UpdateRecord(domain string, record *dnstypes.Record,
 		args["ttl"] = types.String(newRecord.TTL)
 	}
 	_, err := this.post("/Record.Modify", args)
-	return err
+	return this.WrapError(err, domain, newRecord)
 }
 
 // DeleteRecord 删除记录
@@ -225,7 +225,7 @@ func (this *DNSPodProvider) DeleteRecord(domain string, record *dnstypes.Record)
 		"record_id": record.Id,
 	})
 
-	return err
+	return this.WrapError(err, domain, record)
 }
 
 // 发送请求

internal/dnsclients/provider_huawei_dns.go

@@ -1331,7 +1331,7 @@ func (this *HuaweiDNSProvider) QueryRecord(domain string, name string, recordTyp
 func (this *HuaweiDNSProvider) AddRecord(domain string, newRecord *dnstypes.Record) error {
 	zoneId, err := this.findZoneIdWithDomain(domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	var resp = new(huaweidns.ZonesCreateRecordSetResponse)
@@ -1354,7 +1354,7 @@ func (this *HuaweiDNSProvider) AddRecord(domain string, newRecord *dnstypes.Reco
 		"ttl":         ttl,
 	}, resp)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	newRecord.Id = resp.Id + "@" + newRecord.Value
@@ -1366,7 +1366,7 @@ func (this *HuaweiDNSProvider) AddRecord(domain string, newRecord *dnstypes.Reco
 func (this *HuaweiDNSProvider) UpdateRecord(domain string, record *dnstypes.Record, newRecord *dnstypes.Record) error {
 	zoneId, err := this.findZoneIdWithDomain(domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	var recordId string
@@ -1397,7 +1397,7 @@ func (this *HuaweiDNSProvider) UpdateRecord(domain string, record *dnstypes.Reco
 		"ttl":         ttl,
 	}, resp)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	return nil
@@ -1407,7 +1407,7 @@ func (this *HuaweiDNSProvider) UpdateRecord(domain string, record *dnstypes.Reco
 func (this *HuaweiDNSProvider) DeleteRecord(domain string, record *dnstypes.Record) error {
 	zoneId, err := this.findZoneIdWithDomain(domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, record)
 	}
 
 	var recordId string
@@ -1421,7 +1421,7 @@ func (this *HuaweiDNSProvider) DeleteRecord(domain string, record *dnstypes.Reco
 	var resp = new(huaweidns.ZonesDeleteRecordSetResponse)
 	err = this.doAPI(http.MethodDelete, "/v2.1/zones/"+zoneId+"/recordsets/"+recordId, map[string]string{}, maps.Map{}, resp)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, record)
 	}
 
 	return nil

internal/dnsclients/provider_local_edge_dns.go

@@ -17,6 +17,8 @@ import (
 type LocalEdgeDNSProvider struct {
 	clusterId int64 // 集群ID
 	ttl       int32 // TTL
+
+	BaseProvider
 }
 
 // Auth 认证
@@ -193,10 +195,10 @@ func (this *LocalEdgeDNSProvider) AddRecord(domain string, newRecord *dnstypes.R
 	var tx *dbs.Tx
 	domainId, err := nameservers.SharedNSDomainDAO.FindDomainIdWithName(tx, this.clusterId, domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 	if domainId == 0 {
-		return errors.New("can not find domain '" + domain + "'")
+		return this.WrapError(errors.New("can not find domain '"+domain+"'"), domain, newRecord)
 	}
 
 	var routeIds = []string{}
@@ -209,7 +211,7 @@ func (this *LocalEdgeDNSProvider) AddRecord(domain string, newRecord *dnstypes.R
 	}
 	_, err = nameservers.SharedNSRecordDAO.CreateRecord(tx, domainId, "", newRecord.Name, newRecord.Type, newRecord.Value, newRecord.TTL, routeIds)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 
 	return nil
@@ -220,7 +222,7 @@ func (this *LocalEdgeDNSProvider) UpdateRecord(domain string, record *dnstypes.R
 	var tx *dbs.Tx
 	domainId, err := nameservers.SharedNSDomainDAO.FindDomainIdWithName(tx, this.clusterId, domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, newRecord)
 	}
 	if domainId == 0 {
 		return errors.New("can not find domain '" + domain + "'")
@@ -238,17 +240,17 @@ func (this *LocalEdgeDNSProvider) UpdateRecord(domain string, record *dnstypes.R
 	if len(record.Id) > 0 {
 		err = nameservers.SharedNSRecordDAO.UpdateRecord(tx, types.Int64(record.Id), "", newRecord.Name, newRecord.Type, newRecord.Value, newRecord.TTL, routeIds, true)
 		if err != nil {
-			return err
+			return this.WrapError(err, domain, newRecord)
 		}
 	} else {
 		realRecord, err := nameservers.SharedNSRecordDAO.FindEnabledRecordWithName(tx, domainId, record.Name, record.Type)
 		if err != nil {
-			return err
+			return this.WrapError(err, domain, newRecord)
 		}
 		if realRecord != nil {
 			err = nameservers.SharedNSRecordDAO.UpdateRecord(tx, types.Int64(realRecord.Id), "", newRecord.Name, newRecord.Type, newRecord.Value, newRecord.TTL, routeIds, true)
 			if err != nil {
-				return err
+				return this.WrapError(err, domain, newRecord)
 			}
 		}
 	}
@@ -261,7 +263,7 @@ func (this *LocalEdgeDNSProvider) DeleteRecord(domain string, record *dnstypes.R
 	var tx *dbs.Tx
 	domainId, err := nameservers.SharedNSDomainDAO.FindDomainIdWithName(tx, this.clusterId, domain)
 	if err != nil {
-		return err
+		return this.WrapError(err, domain, record)
 	}
 	if domainId == 0 {
 		return errors.New("can not find domain '" + domain + "'")
@@ -270,17 +272,17 @@ func (this *LocalEdgeDNSProvider) DeleteRecord(domain string, record *dnstypes.R
 	if len(record.Id) > 0 {
 		err = nameservers.SharedNSRecordDAO.DisableNSRecord(tx, types.Int64(record.Id))
 		if err != nil {
-			return err
+			return this.WrapError(err, domain, record)
 		}
 	} else {
 		realRecord, err := nameservers.SharedNSRecordDAO.FindEnabledRecordWithName(tx, domainId, record.Name, record.Type)
 		if err != nil {
-			return err
+			return this.WrapError(err, domain, record)
 		}
 		if realRecord != nil {
 			err = nameservers.SharedNSRecordDAO.DisableNSRecord(tx, types.Int64(realRecord.Id))
 			if err != nil {
-				return err
+				return this.WrapError(err, domain, record)
 			}
 		}
 	}

internal/nodes/api_node.go

@@ -15,7 +15,6 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/rpc"
 	"github.com/TeaOSLab/EdgeAPI/internal/setup"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
-	"github.com/go-yaml/yaml"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/lists"
@@ -26,6 +25,7 @@ import (
 	"github.com/iwind/gosock/pkg/gosock"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"log"
 	"net"
@@ -37,6 +37,9 @@ import (
 	"strconv"
 	"sync"
 	"time"
+
+	// grpc decompression
+	_ "google.golang.org/grpc/encoding/gzip"
 )
 
 var sharedAPIConfig *configs.APIConfig = nil
@@ -276,7 +279,7 @@ func (this *APINode) autoUpgrade() error {
 	if err != nil {
 		return errors.New("decode database config failed: " + err.Error())
 	}
-	dbConfig := config.DBs[Tea.Env]
+	var dbConfig = config.DBs[Tea.Env]
 	db, err := dbs.NewInstanceFromConfig(dbConfig)
 	if err != nil {
 		return errors.New("load database failed: " + err.Error())
@@ -287,8 +290,8 @@ func (this *APINode) autoUpgrade() error {
 	}
 	if one != nil {
 		// 如果是同样的版本，则直接认为是最新版本
-		version := one.GetString("version")
-		if stringutil.VersionCompare(version, teaconst.Version) >= 0 {
+		var version = one.GetString("version")
+		if stringutil.VersionCompare(version, setup.ComposeSQLVersion()) >= 0 {
 			return nil
 		}
 	}

internal/nodes/node_status_executor.go

@@ -9,8 +9,8 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/iwind/TeaGo/lists"
-	"github.com/shirou/gopsutil/cpu"
-	"github.com/shirou/gopsutil/disk"
+	"github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v3/disk"
 	"os"
 	"runtime"
 	"strings"

internal/nodes/node_status_executor_test.go

@@ -1,7 +1,7 @@
 package nodes
 
 import (
-	"github.com/shirou/gopsutil/cpu"
+	"github.com/shirou/gopsutil/v3/cpu"
 	"testing"
 	"time"
 )

internal/nodes/node_status_executor_unix.go

@@ -4,8 +4,8 @@ package nodes
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
-	"github.com/shirou/gopsutil/load"
-	"github.com/shirou/gopsutil/mem"
+	"github.com/shirou/gopsutil/v3/load"
+	"github.com/shirou/gopsutil/v3/mem"
 )
 
 // 更新内存

internal/nodes/node_status_executor_windows.go

@@ -4,8 +4,8 @@ package nodes
 
 import (
 	"context"
-	"github.com/shirou/gopsutil/cpu"
-	"github.com/shirou/gopsutil/mem"
+	"github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v3/mem"
 	"math"
 	"sync"
 	"time"

internal/rpc/services/service_admin.go

@@ -9,14 +9,12 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/rpc/tasks"
 	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
-	"github.com/iwind/TeaGo/dbs"
-	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
 )
@@ -743,7 +741,11 @@ func (this *AdminService) ComposeAdminDashboard(ctx context.Context, req *pb.Com
 
 	// 域名排行
 	this.BeginTag(ctx, "SharedServerDomainHourlyStatDAO.FindTopDomainStats")
-	topDomainStats, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStats(tx, hourFrom, hourTo, 10)
+	var topDomainStats []*stats.ServerDomainHourlyStat
+	topDomainStatsCache, ok := tasks.SharedCacheTaskManager.GetGlobalTopDomains()
+	if ok {
+		topDomainStats = topDomainStatsCache.([]*stats.ServerDomainHourlyStat)
+	}
 	this.EndTag(ctx, "SharedServerDomainHourlyStatDAO.FindTopDomainStats")
 	if err != nil {
 		return nil, err
@@ -816,7 +818,11 @@ func (this *AdminService) ComposeAdminDashboard(ctx context.Context, req *pb.Com
 
 	// 指标数据
 	this.BeginTag(ctx, "findMetricDataCharts")
-	pbCharts, err := this.findMetricDataCharts(tx)
+	var pbCharts []*pb.MetricDataChart
+	pbChartsCache, ok := tasks.SharedCacheTaskManager.Get(tasks.CacheKeyFindAllMetricDataCharts)
+	if ok {
+		pbCharts = pbChartsCache.([]*pb.MetricDataChart)
+	}
 	this.EndTag(ctx, "findMetricDataCharts")
 	if err != nil {
 		return nil, err
@@ -839,111 +845,3 @@ func (this *AdminService) UpdateAdminTheme(ctx context.Context, req *pb.UpdateAd
 	}
 	return this.Success()
 }
-
-// 查找集群、节点和服务的指标数据
-func (this *AdminService) findMetricDataCharts(tx *dbs.Tx) (result []*pb.MetricDataChart, err error) {
-	// 集群指标
-	items, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, serverconfigs.MetricItemCategoryHTTP, nil)
-	if err != nil {
-		return nil, err
-	}
-	var pbMetricCharts = []*pb.MetricDataChart{}
-	for _, item := range items {
-		var itemId = int64(item.Id)
-		charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, chart := range charts {
-			if chart.IsOn == 0 {
-				continue
-			}
-
-			var pbChart = &pb.MetricChart{
-				Id:         int64(chart.Id),
-				Name:       chart.Name,
-				Type:       chart.Type,
-				WidthDiv:   chart.WidthDiv,
-				ParamsJSON: nil,
-				IsOn:       chart.IsOn == 1,
-				MaxItems:   types.Int32(chart.MaxItems),
-				MetricItem: &pb.MetricItem{
-					Id:         itemId,
-					PeriodUnit: item.PeriodUnit,
-					Period:     types.Int32(item.Period),
-					Name:       item.Name,
-					Value:      item.Value,
-					Category:   item.Category,
-					Keys:       item.DecodeKeys(),
-					Code:       item.Code,
-					IsOn:       item.IsOn == 1,
-				},
-			}
-			var pbStats = []*pb.MetricStat{}
-			switch chart.Type {
-			case serverconfigs.MetricChartTypeTimeLine:
-				itemStats, err := models.SharedMetricStatDAO.FindLatestItemStats(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				if err != nil {
-					return nil, err
-				}
-
-				for _, stat := range itemStats {
-					// 当前时间总和
-					count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
-					if err != nil {
-						return nil, err
-					}
-
-					pbStats = append(pbStats, &pb.MetricStat{
-						Id:          int64(stat.Id),
-						Hash:        stat.Hash,
-						ServerId:    0,
-						ItemId:      0,
-						Keys:        stat.DecodeKeys(),
-						Value:       types.Float32(stat.Value),
-						Time:        stat.Time,
-						Version:     0,
-						NodeCluster: nil,
-						Node:        nil,
-						Server:      nil,
-						SumCount:    count,
-						SumTotal:    total,
-					})
-				}
-			default:
-				itemStats, err := models.SharedMetricStatDAO.FindItemStatsAtLastTime(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				if err != nil {
-					return nil, err
-				}
-				for _, stat := range itemStats {
-					count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
-					if err != nil {
-						return nil, err
-					}
-
-					pbStats = append(pbStats, &pb.MetricStat{
-						Id:          int64(stat.Id),
-						Hash:        stat.Hash,
-						ServerId:    0,
-						ItemId:      0,
-						Keys:        stat.DecodeKeys(),
-						Value:       types.Float32(stat.Value),
-						Time:        stat.Time,
-						Version:     0,
-						NodeCluster: nil,
-						Node:        nil,
-						Server:      nil,
-						SumCount:    count,
-						SumTotal:    total,
-					})
-				}
-			}
-			pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
-				MetricChart: pbChart,
-				MetricStats: pbStats,
-			})
-		}
-	}
-	return pbMetricCharts, nil
-}

internal/rpc/services/service_db.go

@@ -9,12 +9,12 @@ import (
 	"strings"
 )
 
-// 数据库相关服务
+// DBService 数据库相关服务
 type DBService struct {
 	BaseService
 }
 
-// 获取所有表信息
+// FindAllDBTables 获取所有表信息
 func (this *DBService) FindAllDBTables(ctx context.Context, req *pb.FindAllDBTablesRequest) (*pb.FindAllDBTablesResponse, error) {
 	_, err := this.ValidateAdmin(ctx, 0)
 	if err != nil {
@@ -60,7 +60,7 @@ func (this *DBService) FindAllDBTables(ctx context.Context, req *pb.FindAllDBTab
 	return &pb.FindAllDBTablesResponse{DbTables: pbTables}, nil
 }
 
-// 删除表
+// DeleteDBTable 删除表
 func (this *DBService) DeleteDBTable(ctx context.Context, req *pb.DeleteDBTableRequest) (*pb.RPCSuccess, error) {
 	_, err := this.ValidateAdmin(ctx, 0)
 	if err != nil {
@@ -84,7 +84,7 @@ func (this *DBService) DeleteDBTable(ctx context.Context, req *pb.DeleteDBTableR
 	return this.Success()
 }
 
-// 清空表
+// TruncateDBTable 清空表
 func (this *DBService) TruncateDBTable(ctx context.Context, req *pb.TruncateDBTableRequest) (*pb.RPCSuccess, error) {
 	_, err := this.ValidateAdmin(ctx, 0)
 	if err != nil {

internal/rpc/services/service_firewall.go

@@ -12,6 +12,7 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sort"
 	"strconv"
 	"time"
 )
@@ -118,7 +119,7 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
 		return nil, err
 	}
 	{
-		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "log", dayFrom, day)
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, []string{"log", "tag"}, dayFrom, day)
 		if err != nil {
 			return nil, err
 		}
@@ -131,7 +132,7 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
 		}
 	}
 	{
-		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "captcha", dayFrom, day)
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, []string{"captcha"}, dayFrom, day)
 		if err != nil {
 			return nil, err
 		}
@@ -144,7 +145,7 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
 		}
 	}
 	{
-		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "block", dayFrom, day)
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, []string{"block", "page"}, dayFrom, day)
 		if err != nil {
 			return nil, err
 		}
@@ -158,10 +159,14 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
 	}
 
 	// 规则分组
-	groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, 0, 0, dayFrom, day, 0, 10)
+	var today = timeutil.Format("Ymd")
+	groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, 0, 0, today, today, 0, 20)
 	if err != nil {
 		return nil, err
 	}
+
+	// 合并同名
+	var groupNamedStatsMap = map[string]*stats.ServerHTTPFirewallDailyStat{} // name => *ServerHTTPFirewallDailyStat
 	for _, stat := range groupStats {
 		ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId))
 		if err != nil {
@@ -171,11 +176,26 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
 			continue
 		}
 
+		namedStat, ok := groupNamedStatsMap[ruleGroupName]
+		if ok {
+			namedStat.Count += stat.Count
+		} else {
+			groupNamedStatsMap[ruleGroupName] = stat
+		}
+	}
+
+	for ruleGroupName, stat := range groupNamedStatsMap {
 		result.HttpFirewallRuleGroups = append(result.HttpFirewallRuleGroups, &pb.ComposeFirewallGlobalBoardResponse_HTTPFirewallRuleGroupStat{
 			HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName},
 			Count:                 int64(stat.Count),
 		})
 	}
+	sort.Slice(result.HttpFirewallRuleGroups, func(i, j int) bool {
+		return result.HttpFirewallRuleGroups[i].Count > result.HttpFirewallRuleGroups[j].Count
+	})
+	if len(result.HttpFirewallRuleGroups) > 10 {
+		result.HttpFirewallRuleGroups = result.HttpFirewallRuleGroups[:10]
+	}
 
 	// 节点排行
 	topNodeStats, err := stats.SharedNodeTrafficHourlyStatDAO.FindTopNodeStatsWithAttack(tx, "node", hourFrom, hourTo, 10)

internal/rpc/services/service_http_access_log.go

@@ -72,7 +72,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
 		}
 	}
 
-	accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
+	accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/services/service_http_cache_policy.go

@@ -46,7 +46,7 @@ func (this *HTTPCachePolicyService) CreateHTTPCachePolicy(ctx context.Context, r
 
 	tx := this.NullTx()
 
-	policyId, err := models.SharedHTTPCachePolicyDAO.CreateCachePolicy(tx, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON)
+	policyId, err := models.SharedHTTPCachePolicyDAO.CreateCachePolicy(tx, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache)
 	if err != nil {
 		return nil, err
 	}
@@ -63,7 +63,7 @@ func (this *HTTPCachePolicyService) UpdateHTTPCachePolicy(ctx context.Context, r
 
 	tx := this.NullTx()
 
-	err = models.SharedHTTPCachePolicyDAO.UpdateCachePolicy(tx, req.HttpCachePolicyId, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON)
+	err = models.SharedHTTPCachePolicyDAO.UpdateCachePolicy(tx, req.HttpCachePolicyId, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache)
 	if err != nil {
 		return nil, err
 	}
@@ -99,7 +99,7 @@ func (this *HTTPCachePolicyService) CountAllEnabledHTTPCachePolicies(ctx context
 
 	tx := this.NullTx()
 
-	count, err := models.SharedHTTPCachePolicyDAO.CountAllEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword)
+	count, err := models.SharedHTTPCachePolicyDAO.CountAllEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword, req.Type)
 	if err != nil {
 		return nil, err
 	}
@@ -116,7 +116,7 @@ func (this *HTTPCachePolicyService) ListEnabledHTTPCachePolicies(ctx context.Con
 
 	tx := this.NullTx()
 
-	cachePolicies, err := models.SharedHTTPCachePolicyDAO.ListEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword, req.Offset, req.Size)
+	cachePolicies, err := models.SharedHTTPCachePolicyDAO.ListEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword, req.Type, req.Offset, req.Size)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/services/service_node.go

@@ -578,6 +578,7 @@ func (this *NodeService) FindEnabledNode(ctx context.Context, req *pb.FindEnable
 		NodeRegion:             pbRegion,
 		MaxCacheDiskCapacity:   pbMaxCacheDiskCapacity,
 		MaxCacheMemoryCapacity: pbMaxCacheMemoryCapacity,
+		CacheDiskDir:           node.CacheDiskDir,
 	}}, nil
 }
 
@@ -1570,7 +1571,7 @@ func (this *NodeService) UpdateNodeCache(ctx context.Context, req *pb.UpdateNode
 		}
 	}
 
-	err = models.SharedNodeDAO.UpdateNodeCache(tx, req.NodeId, maxCacheDiskCapacityJSON, maxCacheMemoryCapacityJSON)
+	err = models.SharedNodeDAO.UpdateNodeCache(tx, req.NodeId, maxCacheDiskCapacityJSON, maxCacheMemoryCapacityJSON, req.CacheDiskDir)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/services/service_node_cluster.go

@@ -298,6 +298,7 @@ func (this *NodeClusterService) ListEnabledNodeClusters(ctx context.Context, req
 			DnsDomainId: int64(cluster.DnsDomainId),
 			IsOn:        cluster.IsOn == 1,
 			TimeZone:    cluster.TimeZone,
+			IsPinned:    cluster.IsPinned == 1,
 		})
 	}
 
@@ -1026,3 +1027,19 @@ func (this *NodeClusterService) FindEnabledNodeClusterConfigInfo(ctx context.Con
 
 	return result, nil
 }
+
+// UpdateNodeClusterPinned 设置集群是否置顶
+func (this *NodeClusterService) UpdateNodeClusterPinned(ctx context.Context, req *pb.UpdateNodeClusterPinnedRequest) (*pb.RPCSuccess, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	err = models.SharedNodeClusterDAO.UpdateClusterIsPinned(tx, req.NodeClusterId, req.IsPinned)
+	if err != nil {
+		return nil, err
+	}
+
+	return this.Success()
+}

internal/rpc/services/service_origin.go

@@ -72,7 +72,7 @@ func (this *OriginService) CreateOrigin(ctx context.Context, req *pb.CreateOrigi
 		}
 	}
 
-	originId, err := models.SharedOriginDAO.CreateOrigin(tx, adminId, userId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains)
+	originId, err := models.SharedOriginDAO.CreateOrigin(tx, adminId, userId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains, req.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -139,7 +139,7 @@ func (this *OriginService) UpdateOrigin(ctx context.Context, req *pb.UpdateOrigi
 		}
 	}
 
-	err = models.SharedOriginDAO.UpdateOrigin(tx, req.OriginId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains)
+	err = models.SharedOriginDAO.UpdateOrigin(tx, req.OriginId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains, req.Host)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/services/service_reverse_proxy.go

@@ -216,7 +216,7 @@ func (this *ReverseProxyService) UpdateReverseProxy(ctx context.Context, req *pb
 		}
 	}
 
-	err = models.SharedReverseProxyDAO.UpdateReverseProxy(tx, req.ReverseProxyId, types.Int8(req.RequestHostType), req.RequestHost, req.RequestURI, req.StripPrefix, req.AutoFlush, req.AddHeaders, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, req.ProxyProtocolJSON)
+	err = models.SharedReverseProxyDAO.UpdateReverseProxy(tx, req.ReverseProxyId, types.Int8(req.RequestHostType), req.RequestHost, req.RequestURI, req.StripPrefix, req.AutoFlush, req.AddHeaders, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, req.ProxyProtocolJSON, req.FollowRedirects)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/services/service_server.go

@@ -42,7 +42,7 @@ func (this *ServerService) CreateServer(ctx context.Context, req *pb.CreateServe
 				return nil, err
 			}
 			if httpsConfig.SSLPolicyRef != nil && httpsConfig.SSLPolicyRef.SSLPolicyId > 0 {
-				err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, httpsConfig.SSLPolicyRef.SSLPolicyId, userId)
+				err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, userId, httpsConfig.SSLPolicyRef.SSLPolicyId)
 				if err != nil {
 					return nil, err
 				}
@@ -57,7 +57,7 @@ func (this *ServerService) CreateServer(ctx context.Context, req *pb.CreateServe
 				return nil, err
 			}
 			if tlsConfig.SSLPolicyRef != nil && tlsConfig.SSLPolicyRef.SSLPolicyId > 0 {
-				err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, tlsConfig.SSLPolicyRef.SSLPolicyId, userId)
+				err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, userId, tlsConfig.SSLPolicyRef.SSLPolicyId)
 				if err != nil {
 					return nil, err
 				}

internal/rpc/services/service_server_http_firewall_daily_stat.go

@@ -8,6 +8,7 @@ import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sort"
 	"time"
 )
 
@@ -90,10 +91,13 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
 	}
 
 	// 规则分组
-	groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, userId, req.ServerId, monthFrom, monthTo, 0, 10)
+	groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, userId, req.ServerId, monthFrom, monthTo, 0, 20)
 	if err != nil {
 		return nil, err
 	}
+
+	// 合并同名
+	var groupNamedStatsMap = map[string]*stats.ServerHTTPFirewallDailyStat{} // name => *ServerHTTPFirewallDailyStat
 	for _, stat := range groupStats {
 		ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId))
 		if err != nil {
@@ -103,11 +107,26 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
 			continue
 		}
 
+		namedStat, ok := groupNamedStatsMap[ruleGroupName]
+		if ok {
+			namedStat.Count += stat.Count
+		} else {
+			groupNamedStatsMap[ruleGroupName] = stat
+		}
+	}
+
+	for ruleGroupName, stat := range groupNamedStatsMap {
 		resp.HttpFirewallRuleGroups = append(resp.HttpFirewallRuleGroups, &pb.ComposeServerHTTPFirewallDashboardResponse_HTTPFirewallRuleGroupStat{
 			HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName},
 			Count:                 int64(stat.Count),
 		})
 	}
+	sort.Slice(resp.HttpFirewallRuleGroups, func(i, j int) bool {
+		return resp.HttpFirewallRuleGroups[i].Count > resp.HttpFirewallRuleGroups[j].Count
+	})
+	if len(resp.HttpFirewallRuleGroups) > 10 {
+		resp.HttpFirewallRuleGroups = resp.HttpFirewallRuleGroups[:10]
+	}
 
 	// 每日趋势
 	dayBefore := timeutil.Format("Ymd", date.AddDate(0, 0, -14))
@@ -116,7 +135,7 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
 		return nil, err
 	}
 	{
-		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "log", dayBefore, day)
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, []string{"log", "tag"}, dayBefore, day)
 		if err != nil {
 			return nil, err
 		}
@@ -129,7 +148,7 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
 		}
 	}
 	{
-		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "block", dayBefore, day)
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, []string{"block", "page"}, dayBefore, day)
 		if err != nil {
 			return nil, err
 		}
@@ -142,7 +161,7 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
 		}
 	}
 	{
-		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "captcha", dayBefore, day)
+		statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, []string{"captcha"}, dayBefore, day)
 		if err != nil {
 			return nil, err
 		}

internal/rpc/services/service_server_stat_board.go

@@ -9,12 +9,11 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/rpc/tasks"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/iwind/TeaGo/dbs"
-	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"time"
@@ -208,11 +207,12 @@ func (this *ServerStatBoardService) ComposeServerStatNodeClusterBoard(ctx contex
 		})
 	}
 
-	charts, err := this.findNodeClusterMetricDataCharts(tx, req.NodeClusterId, 0, 0, serverconfigs.MetricItemCategoryHTTP)
-	if err != nil {
-		return nil, err
+	var pbCharts []*pb.MetricDataChart
+	charts, ok := tasks.SharedCacheTaskManager.GetCluster(tasks.CacheKeyFindNodeClusterMetricDataCharts, req.NodeClusterId, serverconfigs.MetricItemCategoryHTTP)
+	if ok {
+		pbCharts = charts.([]*pb.MetricDataChart)
 	}
-	result.MetricDataCharts = charts
+	result.MetricDataCharts = pbCharts
 
 	return result, nil
 }
@@ -443,11 +443,12 @@ func (this *ServerStatBoardService) ComposeServerStatNodeBoard(ctx context.Conte
 
 	// 指标
 	var clusterId = int64(node.ClusterId)
-	charts, err := this.findNodeClusterMetricDataCharts(tx, clusterId, req.NodeId, 0, serverconfigs.MetricItemCategoryHTTP)
-	if err != nil {
-		return nil, err
+	var pbCharts []*pb.MetricDataChart
+	charts, ok := tasks.SharedCacheTaskManager.GetNode(tasks.CacheKeyFindNodeMetricDataCharts, clusterId, req.NodeId, serverconfigs.MetricItemCategoryHTTP)
+	if ok {
+		pbCharts = charts.([]*pb.MetricDataChart)
 	}
-	result.MetricDataCharts = charts
+	result.MetricDataCharts = pbCharts
 
 	return result, nil
 }
@@ -560,182 +561,13 @@ func (this *ServerStatBoardService) ComposeServerStatBoard(ctx context.Context,
 	case serverconfigs.ServerTypeUDPProxy:
 		metricCategory = serverconfigs.MetricItemCategoryUDP
 	}
-	charts, err := this.findNodeClusterMetricDataCharts(tx, clusterId, 0, req.ServerId, metricCategory)
-	if err != nil {
-		return nil, err
+
+	var pbCharts []*pb.MetricDataChart
+	charts, ok := tasks.SharedCacheTaskManager.GetServer(tasks.CacheKeyFindServerMetricDataCharts, clusterId, req.ServerId, metricCategory)
+	if ok {
+		pbCharts = charts.([]*pb.MetricDataChart)
 	}
-	result.MetricDataCharts = charts
+	result.MetricDataCharts = pbCharts
 
 	return result, nil
 }
-
-// 查找集群、节点和服务的指标数据
-func (this *ServerStatBoardService) findNodeClusterMetricDataCharts(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, category string) (result []*pb.MetricDataChart, err error) {
-	// 集群指标
-	clusterMetricItems, err := models.SharedNodeClusterMetricItemDAO.FindAllClusterItems(tx, clusterId, category)
-	if err != nil {
-		return nil, err
-	}
-	var pbMetricCharts = []*pb.MetricDataChart{}
-	var metricItemIds = []int64{}
-	var items = []*models.MetricItem{}
-	for _, clusterMetricItem := range clusterMetricItems {
-		if clusterMetricItem.IsOn != 1 {
-			continue
-		}
-		var itemId = int64(clusterMetricItem.ItemId)
-		item, err := models.SharedMetricItemDAO.FindEnabledMetricItem(tx, itemId)
-		if err != nil {
-			return nil, err
-		}
-		if item == nil || item.IsOn == 0 {
-			continue
-		}
-		items = append(items, item)
-		metricItemIds = append(metricItemIds, itemId)
-	}
-
-	publicMetricItems, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, category, nil)
-	if err != nil {
-		return nil, err
-	}
-	for _, item := range publicMetricItems {
-		if item.IsOn != 1 {
-			continue
-		}
-		if lists.ContainsInt64(metricItemIds, int64(item.Id)) {
-			continue
-		}
-		items = append(items, item)
-	}
-
-	for _, item := range items {
-		var itemId = int64(item.Id)
-		charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, chart := range charts {
-			if chart.IsOn == 0 {
-				continue
-			}
-
-			var pbChart = &pb.MetricChart{
-				Id:         int64(chart.Id),
-				Name:       chart.Name,
-				Type:       chart.Type,
-				WidthDiv:   chart.WidthDiv,
-				ParamsJSON: nil,
-				IsOn:       chart.IsOn == 1,
-				MaxItems:   types.Int32(chart.MaxItems),
-				MetricItem: &pb.MetricItem{
-					Id:         itemId,
-					PeriodUnit: item.PeriodUnit,
-					Period:     types.Int32(item.Period),
-					Name:       item.Name,
-					Value:      item.Value,
-					Category:   item.Category,
-					Keys:       item.DecodeKeys(),
-					Code:       item.Code,
-					IsOn:       item.IsOn == 1,
-				},
-			}
-			var pbStats = []*pb.MetricStat{}
-			switch chart.Type {
-			case serverconfigs.MetricChartTypeTimeLine:
-				var itemStats []*models.MetricStat
-				if serverId > 0 {
-					itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithServerId(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				} else if nodeId > 0 {
-					itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithNodeId(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				} else {
-					itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithClusterId(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				}
-				if err != nil {
-					return nil, err
-				}
-
-				for _, stat := range itemStats {
-					// 当前时间总和
-					var count int64
-					var total float32
-					if serverId > 0 {
-						count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
-					} else if nodeId > 0 {
-						count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
-					} else {
-						count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
-					}
-					if err != nil {
-						return nil, err
-					}
-
-					pbStats = append(pbStats, &pb.MetricStat{
-						Id:          int64(stat.Id),
-						Hash:        stat.Hash,
-						ServerId:    0,
-						ItemId:      0,
-						Keys:        stat.DecodeKeys(),
-						Value:       types.Float32(stat.Value),
-						Time:        stat.Time,
-						Version:     0,
-						NodeCluster: nil,
-						Node:        nil,
-						Server:      nil,
-						SumCount:    count,
-						SumTotal:    total,
-					})
-				}
-			default:
-				var itemStats []*models.MetricStat
-				if serverId > 0 {
-					itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithServerIdAndLastTime(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				} else if nodeId > 0 {
-					itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithNodeIdAndLastTime(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				} else {
-					itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithClusterIdAndLastTime(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
-				}
-				if err != nil {
-					return nil, err
-				}
-				for _, stat := range itemStats {
-					// 当前时间总和
-					var count int64
-					var total float32
-					if serverId > 0 {
-						count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
-					} else if nodeId > 0 {
-						count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
-					} else {
-						count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
-					}
-					if err != nil {
-						return nil, err
-					}
-
-					pbStats = append(pbStats, &pb.MetricStat{
-						Id:          int64(stat.Id),
-						Hash:        stat.Hash,
-						ServerId:    0,
-						ItemId:      0,
-						Keys:        stat.DecodeKeys(),
-						Value:       types.Float32(stat.Value),
-						Time:        stat.Time,
-						Version:     0,
-						NodeCluster: nil,
-						Node:        nil,
-						Server:      nil,
-						SumCount:    count,
-						SumTotal:    total,
-					})
-				}
-			}
-			pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
-				MetricChart: pbChart,
-				MetricStats: pbStats,
-			})
-		}
-	}
-	return pbMetricCharts, nil
-}

internal/rpc/services/service_ssl_cert.go

@@ -7,6 +7,7 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/acme"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
+	"github.com/iwind/TeaGo/types"
 )
 
 // SSLCertService SSL证书相关服务
@@ -180,3 +181,132 @@ func (this *SSLCertService) ListSSLCerts(ctx context.Context, req *pb.ListSSLCer
 	}
 	return &pb.ListSSLCertsResponse{SslCertsJSON: certConfigsJSON}, nil
 }
+
+// CountAllSSLCertsWithOCSPError 计算有OCSP错误的证书数量
+func (this *SSLCertService) CountAllSSLCertsWithOCSPError(ctx context.Context, req *pb.CountAllSSLCertsWithOCSPErrorRequest) (*pb.RPCCountResponse, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	count, err := models.SharedSSLCertDAO.CountAllSSLCertsWithOCSPError(tx, req.Keyword)
+	if err != nil {
+		return nil, err
+	}
+	return this.SuccessCount(count)
+}
+
+// ListSSLCertsWithOCSPError 列出有OCSP错误的证书
+func (this *SSLCertService) ListSSLCertsWithOCSPError(ctx context.Context, req *pb.ListSSLCertsWithOCSPErrorRequest) (*pb.ListSSLCertsWithOCSPErrorResponse, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	certs, err := models.SharedSSLCertDAO.ListSSLCertsWithOCSPError(tx, req.Keyword, req.Offset, req.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	var pbCerts = []*pb.SSLCert{}
+	for _, cert := range certs {
+		pbCerts = append(pbCerts, &pb.SSLCert{
+			Id:            int64(cert.Id),
+			IsOn:          cert.IsOn == 1,
+			Name:          cert.Name,
+			TimeBeginAt:   types.Int64(cert.TimeBeginAt),
+			TimeEndAt:     types.Int64(cert.TimeEndAt),
+			DnsNames:      cert.DecodeDNSNames(),
+			CommonNames:   cert.DecodeCommonNames(),
+			IsACME:        cert.IsACME == 1,
+			AcmeTaskId:    int64(cert.AcmeTaskId),
+			Ocsp:          []byte(cert.Ocsp),
+			OcspIsUpdated: cert.OcspIsUpdated == 1,
+			OcspError:     cert.OcspError,
+			Description:   cert.Description,
+			IsCA:          cert.IsCA == 1,
+			ServerName:    cert.ServerName,
+			CreatedAt:     int64(cert.CreatedAt),
+			UpdatedAt:     int64(cert.UpdatedAt),
+		})
+	}
+
+	return &pb.ListSSLCertsWithOCSPErrorResponse{
+		SslCerts: pbCerts,
+	}, nil
+}
+
+// IgnoreSSLCertsWithOCSPError 忽略一组OCSP证书错误
+func (this *SSLCertService) IgnoreSSLCertsWithOCSPError(ctx context.Context, req *pb.IgnoreSSLCertsWithOCSPErrorRequest) (*pb.RPCSuccess, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	err = models.SharedSSLCertDAO.IgnoreSSLCertsWithOCSPError(tx, req.SslCertIds)
+	if err != nil {
+		return nil, err
+	}
+	return this.Success()
+}
+
+// ResetSSLCertsWithOCSPError 重置一组证书OCSP错误状态
+func (this *SSLCertService) ResetSSLCertsWithOCSPError(ctx context.Context, req *pb.ResetSSLCertsWithOCSPErrorRequest) (*pb.RPCSuccess, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	err = models.SharedSSLCertDAO.ResetSSLCertsWithOCSPError(tx, req.SslCertIds)
+	if err != nil {
+		return nil, err
+	}
+	return this.Success()
+}
+
+// ResetAllSSLCertsWithOCSPError 重置所有证书OCSP错误状态
+func (this *SSLCertService) ResetAllSSLCertsWithOCSPError(ctx context.Context, req *pb.ResetAllSSLCertsWithOCSPErrorRequest) (*pb.RPCSuccess, error) {
+	_, err := this.ValidateAdmin(ctx, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	err = models.SharedSSLCertDAO.ResetAllSSLCertsWithOCSPError(tx)
+	if err != nil {
+		return nil, err
+	}
+	return this.Success()
+}
+
+// ListUpdatedSSLCertOCSP 读取证书的OCSP
+func (this *SSLCertService) ListUpdatedSSLCertOCSP(ctx context.Context, req *pb.ListUpdatedSSLCertOCSPRequest) (*pb.ListUpdatedSSLCertOCSPResponse, error) {
+	_, err := this.ValidateNode(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var tx = this.NullTx()
+	certs, err := models.SharedSSLCertDAO.ListCertOCSPAfterVersion(tx, req.Version, int64(req.Size))
+	if err != nil {
+		return nil, err
+	}
+
+	var result = []*pb.ListUpdatedSSLCertOCSPResponse_SSLCertOCSP{}
+	for _, cert := range certs {
+		result = append(result, &pb.ListUpdatedSSLCertOCSPResponse_SSLCertOCSP{
+			SslCertId: int64(cert.Id),
+			Data:      []byte(cert.Ocsp),
+			ExpiresAt: int64(cert.OcspExpiresAt),
+			Version:   int64(cert.OcspUpdatedVersion),
+		})
+	}
+
+	return &pb.ListUpdatedSSLCertOCSPResponse{
+		SslCertOCSP: result,
+	}, nil
+}

internal/rpc/services/service_ssl_policy.go

@@ -3,6 +3,7 @@ package services
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
@@ -43,7 +44,7 @@ func (this *SSLPolicyService) CreateSSLPolicy(ctx context.Context, req *pb.Creat
 		// TODO
 	}
 
-	policyId, err := models.SharedSSLPolicyDAO.CreatePolicy(tx, adminId, userId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
+	policyId, err := models.SharedSSLPolicyDAO.CreatePolicy(tx, adminId, userId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.OcspIsOn, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
 	if err != nil {
 		return nil, err
 	}
@@ -62,13 +63,13 @@ func (this *SSLPolicyService) UpdateSSLPolicy(ctx context.Context, req *pb.Updat
 	tx := this.NullTx()
 
 	if userId > 0 {
-		err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, req.SslPolicyId, userId)
+		err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, userId, req.SslPolicyId)
 		if err != nil {
-			return nil, err
+			return nil, errors.New("check ssl policy failed: " + err.Error())
 		}
 	}
 
-	err = models.SharedSSLPolicyDAO.UpdatePolicy(tx, req.SslPolicyId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
+	err = models.SharedSSLPolicyDAO.UpdatePolicy(tx, req.SslPolicyId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.OcspIsOn, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/tasks/cache_task_manager.go

@@ -0,0 +1,494 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/lists"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sync"
+	"time"
+)
+
+var SharedCacheTaskManager = NewCacheTaskManager()
+
+const (
+	// global keys
+
+	CacheKeyFindAllMetricDataCharts = "findAllMetricDataCharts"
+	CacheKeyFindGlobalTopDomains    = "globalFindTopDomains"
+
+	// cluster keys
+
+	CacheKeyFindNodeClusterMetricDataCharts = "findNodeClusterMetricDataCharts"
+
+	// node keys
+
+	CacheKeyFindNodeMetricDataCharts = "findNodeMetricDataCharts"
+
+	// server keys
+
+	CacheKeyFindServerMetricDataCharts = "findServerMetricDataCharts"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			_ = SharedCacheTaskManager.Loop()
+			SharedCacheTaskManager.Start()
+		})
+	})
+}
+
+type CacheTaskManager struct {
+	cacheMap map[string]interface{}
+	locker   sync.Mutex
+
+	nodeTasksTooSlow bool // 记录节点任务是否太慢，如果太慢就不再后台执行
+
+	ticker *time.Ticker
+}
+
+func NewCacheTaskManager() *CacheTaskManager {
+	return &CacheTaskManager{
+		cacheMap: map[string]interface{}{},
+		ticker:   time.NewTicker(5 * time.Minute),
+	}
+}
+
+func (this *CacheTaskManager) Start() {
+	for range this.ticker.C {
+		err := this.Loop()
+		if err != nil {
+			remotelogs.Error("CACHE_TASK_MANAGER", err.Error())
+		}
+	}
+}
+
+func (this *CacheTaskManager) Loop() error {
+	var tx *dbs.Tx
+
+	// Admin看板指标数据
+	{
+		value, err := this.findAllMetricDataCharts(tx)
+		if err != nil {
+			return err
+		}
+		this.locker.Lock()
+		this.cacheMap[CacheKeyFindAllMetricDataCharts] = value
+		this.locker.Unlock()
+	}
+
+	{
+		var hourFrom = timeutil.Format("YmdH", time.Now().Add(-23*time.Hour))
+		var hourTo = timeutil.Format("YmdH")
+
+		value, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStats(tx, hourFrom, hourTo, 10)
+		if err != nil {
+			return err
+		}
+
+		var composedKey = CacheKeyFindGlobalTopDomains
+
+		this.locker.Lock()
+		this.cacheMap[composedKey] = value
+		this.locker.Unlock()
+	}
+
+	// 集群数据
+	clusterIds, err := models.SharedNodeClusterDAO.FindAllEnabledNodeClusterIds(tx)
+	if err != nil {
+		return err
+	}
+	for _, clusterId := range clusterIds {
+		// metric charts
+		{
+			var category = serverconfigs.MetricItemCategoryHTTP
+			var composedKey = CacheKeyFindNodeClusterMetricDataCharts + "@" + types.String(clusterId) + "@" + category
+
+			value, err := this.findNodeClusterMetricDataCharts(tx, clusterId, 0, 0, category)
+			if err != nil {
+				return err
+			}
+			this.locker.Lock()
+			this.cacheMap[composedKey] = value
+			this.locker.Unlock()
+		}
+
+		// nodes
+		var before = time.Now()
+		if !this.nodeTasksTooSlow {
+			nodeIds, err := models.SharedNodeDAO.FindEnabledNodeIdsWithClusterId(tx, clusterId)
+			if err != nil {
+				return err
+			}
+			for _, nodeId := range nodeIds {
+				// metric
+				{
+					var category = serverconfigs.MetricItemCategoryHTTP
+					var composedKey = CacheKeyFindNodeMetricDataCharts + "@" + types.String(clusterId) + "@" + types.String(nodeId) + "@" + category
+
+					value, err := this.findNodeClusterMetricDataCharts(tx, clusterId, nodeId, 0, category)
+					if err != nil {
+						return err
+					}
+					this.locker.Lock()
+					this.cacheMap[composedKey] = value
+					this.locker.Unlock()
+				}
+			}
+		}
+		var cost = time.Since(before).Seconds()
+		if cost > 600 {
+			this.nodeTasksTooSlow = true
+		}
+	}
+
+	return nil
+}
+
+func (this *CacheTaskManager) Get(key string) (value interface{}, ok bool) {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	if key == CacheKeyFindAllMetricDataCharts {
+		value, ok = this.cacheMap[key]
+		return
+	}
+
+	return
+}
+
+func (this *CacheTaskManager) GetCluster(key string, clusterId int64, category string) (value interface{}, ok bool) {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	var composedKey = key + "@" + types.String(clusterId) + "@" + category
+	value, ok = this.cacheMap[composedKey]
+	return
+}
+
+func (this *CacheTaskManager) GetNode(key string, clusterId int64, nodeId int64, category string) (value interface{}, ok bool) {
+	this.locker.Lock()
+	var composedKey = key + "@" + types.String(clusterId) + "@" + types.String(nodeId) + "@" + category
+	value, ok = this.cacheMap[composedKey]
+	this.locker.Unlock()
+
+	if ok {
+		result, err := this.findNodeClusterMetricDataCharts(nil, clusterId, nodeId, 0, category)
+		if err == nil {
+			value = result
+		}
+	}
+
+	return
+}
+
+func (this *CacheTaskManager) GetServer(key string, clusterId int64, serverId int64, category string) (value interface{}, ok bool) {
+	switch key {
+	case CacheKeyFindServerMetricDataCharts:
+		var tx *dbs.Tx
+		v, err := this.findNodeClusterMetricDataCharts(tx, clusterId, 0, serverId, category)
+		if err != nil {
+			return nil, false
+		}
+		return v, true
+	}
+
+	return
+}
+
+func (this *CacheTaskManager) GetGlobalTopDomains() (value interface{}, ok bool) {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	var composedKey = CacheKeyFindGlobalTopDomains
+	value, ok = this.cacheMap[composedKey]
+	return
+}
+
+// 所有集群的指标统计
+func (this *CacheTaskManager) findAllMetricDataCharts(tx *dbs.Tx) (result []*pb.MetricDataChart, err error) {
+	// 集群指标
+	items, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, serverconfigs.MetricItemCategoryHTTP, nil)
+	if err != nil {
+		return nil, err
+	}
+	var pbMetricCharts = []*pb.MetricDataChart{}
+	for _, item := range items {
+		var itemId = int64(item.Id)
+		charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, chart := range charts {
+			if chart.IsOn == 0 {
+				continue
+			}
+
+			var pbChart = &pb.MetricChart{
+				Id:         int64(chart.Id),
+				Name:       chart.Name,
+				Type:       chart.Type,
+				WidthDiv:   chart.WidthDiv,
+				ParamsJSON: nil,
+				IsOn:       chart.IsOn == 1,
+				MaxItems:   types.Int32(chart.MaxItems),
+				MetricItem: &pb.MetricItem{
+					Id:         itemId,
+					PeriodUnit: item.PeriodUnit,
+					Period:     types.Int32(item.Period),
+					Name:       item.Name,
+					Value:      item.Value,
+					Category:   item.Category,
+					Keys:       item.DecodeKeys(),
+					Code:       item.Code,
+					IsOn:       item.IsOn == 1,
+				},
+			}
+			var pbStats = []*pb.MetricStat{}
+			switch chart.Type {
+			case serverconfigs.MetricChartTypeTimeLine:
+				itemStats, err := models.SharedMetricStatDAO.FindLatestItemStats(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				if err != nil {
+					return nil, err
+				}
+
+				for _, stat := range itemStats {
+					// 当前时间总和
+					count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
+					if err != nil {
+						return nil, err
+					}
+
+					pbStats = append(pbStats, &pb.MetricStat{
+						Id:          int64(stat.Id),
+						Hash:        stat.Hash,
+						ServerId:    0,
+						ItemId:      0,
+						Keys:        stat.DecodeKeys(),
+						Value:       types.Float32(stat.Value),
+						Time:        stat.Time,
+						Version:     0,
+						NodeCluster: nil,
+						Node:        nil,
+						Server:      nil,
+						SumCount:    count,
+						SumTotal:    total,
+					})
+				}
+			default:
+				itemStats, err := models.SharedMetricStatDAO.FindItemStatsAtLastTime(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				if err != nil {
+					return nil, err
+				}
+				for _, stat := range itemStats {
+					count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
+					if err != nil {
+						return nil, err
+					}
+
+					pbStats = append(pbStats, &pb.MetricStat{
+						Id:          int64(stat.Id),
+						Hash:        stat.Hash,
+						ServerId:    0,
+						ItemId:      0,
+						Keys:        stat.DecodeKeys(),
+						Value:       types.Float32(stat.Value),
+						Time:        stat.Time,
+						Version:     0,
+						NodeCluster: nil,
+						Node:        nil,
+						Server:      nil,
+						SumCount:    count,
+						SumTotal:    total,
+					})
+				}
+			}
+			pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
+				MetricChart: pbChart,
+				MetricStats: pbStats,
+			})
+		}
+	}
+	return pbMetricCharts, nil
+}
+
+// 某个集群、节点或者服务的指标统计
+func (this *CacheTaskManager) findNodeClusterMetricDataCharts(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, category string) (result []*pb.MetricDataChart, err error) {
+	// 集群指标
+	clusterMetricItems, err := models.SharedNodeClusterMetricItemDAO.FindAllClusterItems(tx, clusterId, category)
+	if err != nil {
+		return nil, err
+	}
+	var pbMetricCharts = []*pb.MetricDataChart{}
+	var metricItemIds = []int64{}
+	var items = []*models.MetricItem{}
+	for _, clusterMetricItem := range clusterMetricItems {
+		if clusterMetricItem.IsOn != 1 {
+			continue
+		}
+		var itemId = int64(clusterMetricItem.ItemId)
+		item, err := models.SharedMetricItemDAO.FindEnabledMetricItem(tx, itemId)
+		if err != nil {
+			return nil, err
+		}
+		if item == nil || item.IsOn == 0 {
+			continue
+		}
+		items = append(items, item)
+		metricItemIds = append(metricItemIds, itemId)
+	}
+
+	publicMetricItems, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, category, nil)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range publicMetricItems {
+		if item.IsOn != 1 {
+			continue
+		}
+		if lists.ContainsInt64(metricItemIds, int64(item.Id)) {
+			continue
+		}
+		items = append(items, item)
+	}
+
+	for _, item := range items {
+		var itemId = int64(item.Id)
+		charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, chart := range charts {
+			if chart.IsOn == 0 {
+				continue
+			}
+
+			var pbChart = &pb.MetricChart{
+				Id:         int64(chart.Id),
+				Name:       chart.Name,
+				Type:       chart.Type,
+				WidthDiv:   chart.WidthDiv,
+				ParamsJSON: nil,
+				IsOn:       chart.IsOn == 1,
+				MaxItems:   types.Int32(chart.MaxItems),
+				MetricItem: &pb.MetricItem{
+					Id:         itemId,
+					PeriodUnit: item.PeriodUnit,
+					Period:     types.Int32(item.Period),
+					Name:       item.Name,
+					Value:      item.Value,
+					Category:   item.Category,
+					Keys:       item.DecodeKeys(),
+					Code:       item.Code,
+					IsOn:       item.IsOn == 1,
+				},
+			}
+			var pbStats = []*pb.MetricStat{}
+			switch chart.Type {
+			case serverconfigs.MetricChartTypeTimeLine:
+				var itemStats []*models.MetricStat
+				if serverId > 0 {
+					itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithServerId(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				} else if nodeId > 0 {
+					itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithNodeId(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				} else {
+					itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithClusterId(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				}
+				if err != nil {
+					return nil, err
+				}
+
+				for _, stat := range itemStats {
+					// 当前时间总和
+					var count int64
+					var total float32
+					if serverId > 0 {
+						count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
+					} else if nodeId > 0 {
+						count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
+					} else {
+						count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
+					}
+					if err != nil {
+						return nil, err
+					}
+
+					pbStats = append(pbStats, &pb.MetricStat{
+						Id:          int64(stat.Id),
+						Hash:        stat.Hash,
+						ServerId:    0,
+						ItemId:      0,
+						Keys:        stat.DecodeKeys(),
+						Value:       types.Float32(stat.Value),
+						Time:        stat.Time,
+						Version:     0,
+						NodeCluster: nil,
+						Node:        nil,
+						Server:      nil,
+						SumCount:    count,
+						SumTotal:    total,
+					})
+				}
+			default:
+				var itemStats []*models.MetricStat
+				if serverId > 0 {
+					itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithServerIdAndLastTime(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				} else if nodeId > 0 {
+					itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithNodeIdAndLastTime(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				} else {
+					itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithClusterIdAndLastTime(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
+				}
+				if err != nil {
+					return nil, err
+				}
+				for _, stat := range itemStats {
+					// 当前时间总和
+					var count int64
+					var total float32
+					if serverId > 0 {
+						count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
+					} else if nodeId > 0 {
+						count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
+					} else {
+						count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
+					}
+					if err != nil {
+						return nil, err
+					}
+
+					pbStats = append(pbStats, &pb.MetricStat{
+						Id:          int64(stat.Id),
+						Hash:        stat.Hash,
+						ServerId:    0,
+						ItemId:      0,
+						Keys:        stat.DecodeKeys(),
+						Value:       types.Float32(stat.Value),
+						Time:        stat.Time,
+						Version:     0,
+						NodeCluster: nil,
+						Node:        nil,
+						Server:      nil,
+						SumCount:    count,
+						SumTotal:    total,
+					})
+				}
+			}
+			pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
+				MetricChart: pbChart,
+				MetricStats: pbStats,
+			})
+		}
+	}
+	return pbMetricCharts, nil
+}

internal/setup/setup.go

@@ -6,11 +6,11 @@ import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/go-yaml/yaml"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/cmd"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/types"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"strconv"

internal/setup/sql_dump.go

@@ -45,7 +45,10 @@ func (this *SQLDump) Dump(db *dbs.DB) (result *SQLDumpResult, err error) {
 	}
 	for _, tableName := range tableNames {
 		// 忽略一些分表
-		if strings.HasPrefix(tableName, "edgeHTTPAccessLogs_") {
+		if strings.HasPrefix(strings.ToLower(tableName), strings.ToLower("edgeHTTPAccessLogs_")) {
+			continue
+		}
+		if strings.HasPrefix(strings.ToLower(tableName), strings.ToLower("edgeNSAccessLogs_")) {
 			continue
 		}
 

internal/setup/sql_executor.go

@@ -2,20 +2,19 @@ package setup
 
 import (
 	"encoding/json"
-	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
 	_ "github.com/go-sql-driver/mysql"
-	"github.com/go-yaml/yaml"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
 	stringutil "github.com/iwind/TeaGo/utils/string"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"time"
 )
@@ -113,7 +112,7 @@ func (this *SQLExecutor) checkData(db *dbs.DB) error {
 	}
 
 	// 更新版本号
-	err = this.updateVersion(db, teaconst.Version)
+	err = this.updateVersion(db, ComposeSQLVersion())
 	if err != nil {
 		return err
 	}

internal/setup/sql_upgrade.go

@@ -69,6 +69,9 @@ var upgradeFuncs = []*upgradeVersion{
 	{
 		"0.4.1", upgradeV0_4_1,
 	},
+	{
+		"0.4.5", upgradeV0_4_5,
+	},
 }
 
 // UpgradeSQLData 升级SQL数据
@@ -590,3 +593,48 @@ func upgradeV0_4_1(db *dbs.DB) error {
 
 	return nil
 }
+
+// v0.4.5
+func upgradeV0_4_5(db *dbs.DB) error {
+	// 升级访问日志自动分表
+	{
+		var dao = models.NewSysSettingDAO()
+		valueJSON, err := dao.ReadSetting(nil, systemconfigs.SettingCodeAccessLogQueue)
+		if err != nil {
+			return err
+		}
+		if len(valueJSON) > 0 {
+			var config = &serverconfigs.AccessLogQueueConfig{}
+			err = json.Unmarshal(valueJSON, config)
+			if err == nil && config.RowsPerTable == 0 {
+				config.EnableAutoPartial = true
+				config.RowsPerTable = 500_000
+				configJSON, err := json.Marshal(config)
+				if err == nil {
+					err = dao.UpdateSetting(nil, systemconfigs.SettingCodeAccessLogQueue, configJSON)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	// 升级一个SQL注入规则
+	{
+		var dao = models.NewHTTPFirewallRuleDAO()
+		ones, _, err := dao.Instance.FindOnes(`SELECT id FROM edgeHTTPFirewallRules WHERE value=?`, "(updatexml|extractvalue|ascii|ord|char|chr|count|concat|rand|floor|substr|length|len|user|database|benchmark|analyse)\\s*\\(")
+		if err != nil {
+			return err
+		}
+		for _, one := range ones {
+			var ruleId = one.GetInt64("id")
+			_, err = dao.Instance.Exec(`UPDATE edgeHTTPFirewallRules SET value=? WHERE id=? LIMIT 1`, `\b(updatexml|extractvalue|ascii|ord|char|chr|count|concat|rand|floor|substr|length|len|user|database|benchmark|analyse)\s*\(.*\)`, ruleId)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

internal/setup/sql_upgrade_test.go

@@ -116,3 +116,20 @@ func TestUpgradeSQLData_v0_4_1(t *testing.T) {
 	}
 	t.Log("ok")
 }
+
+
+func TestUpgradeSQLData_v0_4_5(t *testing.T) {
+	db, err := dbs.NewInstanceFromConfig(&dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = upgradeV0_4_5(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

internal/setup/utils.go

@@ -0,0 +1,20 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package setup
+
+import (
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"strings"
+)
+
+func ComposeSQLVersion() string {
+	var version = teaconst.Version
+	if len(teaconst.SQLVersion) == 0 {
+		return version
+	}
+
+	if strings.Count(version, ".") <= 2 {
+		return version + "." + teaconst.SQLVersion
+	}
+	return version
+}

internal/setup/utils_test.go

@@ -0,0 +1,12 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package setup_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/setup"
+	"testing"
+)
+
+func TestComposeSQLVersion(t *testing.T) {
+	t.Log(setup.ComposeSQLVersion())
+}

internal/tasks/log_task.go

@@ -38,34 +38,17 @@ func (this *LogTask) Run() {
 }
 
 func (this *LogTask) runClean() {
-	ticker := utils.NewTicker(24 * time.Hour)
+	var ticker = utils.NewTicker(24 * time.Hour)
 	for ticker.Wait() {
-		err := this.loopClean(86400)
+		err := this.loopClean()
 		if err != nil {
 			logs.Println("[TASK][LOG]" + err.Error())
 		}
 	}
 }
 
-func (this *LogTask) loopClean(seconds int64) error {
-	// 检查上次运行时间，防止重复运行
-	settingKey := "logTaskCleanLoop"
-	timestamp := time.Now().Unix()
-	c, err := models.SharedSysSettingDAO.CompareInt64Setting(nil, settingKey, timestamp-seconds)
-	if err != nil {
-		return err
-	}
-	if c > 0 {
-		return nil
-	}
-
-	// 记录时间
-	err = models.SharedSysSettingDAO.UpdateSetting(nil, settingKey, []byte(numberutils.FormatInt64(timestamp)))
-	if err != nil {
-		return err
-	}
-
-	configKey := "adminLogConfig"
+func (this *LogTask) loopClean() error {
+	var configKey = "adminLogConfig"
 	valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, configKey)
 	if err != nil {
 		return err
@@ -74,7 +57,7 @@ func (this *LogTask) loopClean(seconds int64) error {
 		return nil
 	}
 
-	config := &systemconfigs.LogConfig{}
+	var config = &systemconfigs.LogConfig{}
 	err = json.Unmarshal(valueJSON, config)
 	if err != nil {
 		return err
@@ -89,7 +72,7 @@ func (this *LogTask) loopClean(seconds int64) error {
 }
 
 func (this *LogTask) runMonitor() {
-	ticker := utils.NewTicker(1 * time.Minute)
+	var ticker = utils.NewTicker(1 * time.Minute)
 	for ticker.Wait() {
 		err := this.loopMonitor(60)
 		if err != nil {
@@ -100,8 +83,8 @@ func (this *LogTask) runMonitor() {
 
 func (this *LogTask) loopMonitor(seconds int64) error {
 	// 检查上次运行时间，防止重复运行
-	settingKey := "logTaskMonitorLoop"
-	timestamp := time.Now().Unix()
+	var settingKey = "logTaskMonitorLoop"
+	var timestamp = time.Now().Unix()
 	c, err := models.SharedSysSettingDAO.CompareInt64Setting(nil, settingKey, timestamp-seconds)
 	if err != nil {
 		return err
@@ -116,7 +99,7 @@ func (this *LogTask) loopMonitor(seconds int64) error {
 		return err
 	}
 
-	configKey := "adminLogConfig"
+	var configKey = "adminLogConfig"
 	valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, configKey)
 	if err != nil {
 		return err
@@ -125,7 +108,7 @@ func (this *LogTask) loopMonitor(seconds int64) error {
 		return nil
 	}
 
-	config := &systemconfigs.LogConfig{}
+	var config = &systemconfigs.LogConfig{}
 	err = json.Unmarshal(valueJSON, config)
 	if err != nil {
 		return err

internal/tasks/log_task_test.go

@@ -9,7 +9,7 @@ func TestLogTask_loopClean(t *testing.T) {
 	dbs.NotifyReady()
 
 	task := NewLogTask()
-	err := task.loopClean(5)
+	err := task.loopClean()
 	if err != nil {
 		t.Fatal(err)
 	}

internal/tasks/server_access_log_cleaner.go

@@ -103,23 +103,18 @@ func (this *ServerAccessLogCleaner) cleanDB(db *dbs.DB, endDay string) error {
 		return errors.New("invalid column names: " + strings.Join(columnNames, ", "))
 	}
 	columnName := columnNames[0]
+	var reg = regexp.MustCompile(`^(?i)(edgeHTTPAccessLogs|edgeNSAccessLogs)_(\d{8})(_\d{4})?$`)
 	for _, one := range ones {
-		tableName := one.GetString(columnName)
+		var tableName = one.GetString(columnName)
 		if len(tableName) == 0 {
 			continue
 		}
-		ok, err := regexp.MatchString(`^(?i)(edgeHTTPAccessLogs|edgeNSAccessLogs)_(\d{8})$`, tableName)
-		if err != nil {
-			return err
-		}
-		if !ok {
+		if !reg.MatchString(tableName) {
 			continue
 		}
-		index := strings.LastIndex(tableName, "_")
-		if index < 0 {
-			continue
-		}
-		day := tableName[index+1:]
+		var matches = reg.FindStringSubmatch(tableName)
+		var day = matches[2]
+
 		if day < endDay {
 			_, err = db.Exec("DROP TABLE " + tableName)
 			if err != nil {

internal/tasks/server_access_log_cleaner_test.go

@@ -1,6 +1,7 @@
-package tasks
+package tasks_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
 	"github.com/iwind/TeaGo/dbs"
 	"testing"
 )
@@ -8,7 +9,7 @@ import (
 func TestServerAccessLogCleaner_Loop(t *testing.T) {
 	dbs.NotifyReady()
 
-	task := NewServerAccessLogCleaner()
+	task := tasks.NewServerAccessLogCleaner()
 	err := task.Loop()
 	if err != nil {
 		t.Fatal(err)

internal/tasks/ssl_cert_update_ocsp_task.go

@@ -0,0 +1,199 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks
+
+import (
+	"bytes"
+	"crypto"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/iwind/TeaGo/dbs"
+	"golang.org/x/crypto/ocsp"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+func init() {
+	dbs.OnReadyDone(func() {
+		goman.New(func() {
+			NewSSLCertUpdateOCSPTask().Start()
+		})
+	})
+}
+
+type SSLCertUpdateOCSPTask struct {
+	ticker     *time.Ticker
+	httpClient *http.Client
+}
+
+func NewSSLCertUpdateOCSPTask() *SSLCertUpdateOCSPTask {
+	return &SSLCertUpdateOCSPTask{
+		ticker:     time.NewTicker(1 * time.Minute),
+		httpClient: utils.SharedHttpClient(5 * time.Second),
+	}
+}
+
+func (this *SSLCertUpdateOCSPTask) Start() {
+	for range this.ticker.C {
+		err := this.Loop(true)
+		if err != nil {
+			remotelogs.Error("SSLCertUpdateOCSPTask", err.Error())
+		}
+	}
+}
+
+func (this *SSLCertUpdateOCSPTask) Loop(checkLock bool) error {
+	if checkLock {
+		ok, err := models.SharedSysLockerDAO.Lock(nil, "ssl_cert_update_ocsp_task", 60-1) // 假设执行时间为1秒
+		if err != nil {
+			return err
+		}
+		if !ok {
+			return nil
+		}
+	}
+
+	var tx *dbs.Tx
+	// TODO 将来可以设置单次任务条数
+	var size int64 = 60
+	var maxTries = 5
+	certs, err := models.SharedSSLCertDAO.ListCertsToUpdateOCSP(tx, maxTries, size)
+	if err != nil {
+		return errors.New("list certs failed: " + err.Error())
+	}
+
+	if len(certs) == 0 {
+		return nil
+	}
+
+	// 锁定
+	for _, cert := range certs {
+		err := models.SharedSSLCertDAO.PrepareCertOCSPUpdating(tx, int64(cert.Id))
+		if err != nil {
+			return errors.New("prepare cert ocsp updating failed: " + err.Error())
+		}
+	}
+
+	for _, cert := range certs {
+		ocspData, expiresAt, err := this.UpdateCertOCSP(cert)
+		var errString = ""
+		var hasErr = false
+		if err != nil {
+			errString = err.Error()
+			hasErr = true
+
+			remotelogs.Warn("SSLCertUpdateOCSPTask", "update ocsp failed: "+errString)
+		}
+		err = models.SharedSSLCertDAO.UpdateCertOCSP(tx, int64(cert.Id), ocspData, expiresAt, hasErr, errString)
+		if err != nil {
+			return errors.New("update ocsp failed: " + err.Error())
+		}
+	}
+
+	return nil
+}
+
+// UpdateCertOCSP 更新单个证书OCSP
+func (this *SSLCertUpdateOCSPTask) UpdateCertOCSP(certOne *models.SSLCert) (ocspData []byte, expiresAt int64, err error) {
+	if certOne.IsCA == 1 || len(certOne.CertData) == 0 || len(certOne.KeyData) == 0 {
+		return
+	}
+
+	keyPair, err := tls.X509KeyPair([]byte(certOne.CertData), []byte(certOne.KeyData))
+	if err != nil {
+		return nil, 0, errors.New("parse certificate failed: " + err.Error())
+	}
+	if len(keyPair.Certificate) == 0 {
+		return nil, 0, nil
+	}
+
+	var certData = keyPair.Certificate[0]
+	cert, err := x509.ParseCertificate(certData)
+	if err != nil {
+		return nil, 0, errors.New("parse certificate block failed: " + err.Error())
+	}
+
+	// 是否已过期
+	var now = time.Now()
+	if cert.NotBefore.After(now) || cert.NotAfter.Before(now) {
+		return nil, 0, nil
+	}
+
+	if len(cert.IssuingCertificateURL) == 0 || len(cert.OCSPServer) == 0 {
+		return nil, 0, nil
+	}
+
+	if len(cert.DNSNames) == 0 {
+		return nil, 0, nil
+	}
+
+	var issuerURL = cert.IssuingCertificateURL[0]
+	var ocspServerURL = cert.OCSPServer[0]
+
+	issuerReq, err := http.NewRequest(http.MethodGet, issuerURL, nil)
+	if err != nil {
+		return nil, 0, errors.New("request issuer certificate failed: " + err.Error())
+	}
+	issuerReq.Header.Set("User-Agent", teaconst.ProductName+"/"+teaconst.Version)
+	issuerResp, err := this.httpClient.Do(issuerReq)
+	if err != nil {
+		return nil, 0, errors.New("request issuer certificate failed: '" + issuerURL + "': " + err.Error())
+	}
+	defer func() {
+		_ = issuerResp.Body.Close()
+	}()
+
+	issuerData, err := ioutil.ReadAll(issuerResp.Body)
+	if err != nil {
+		return nil, 0, errors.New("read issuer certificate failed: '" + issuerURL + "': " + err.Error())
+	}
+	issuerCert, err := x509.ParseCertificate(issuerData)
+	if err != nil {
+		return nil, 0, errors.New("parse issuer certificate failed: '" + issuerURL + "': " + err.Error())
+	}
+
+	buf, err := ocsp.CreateRequest(cert, issuerCert, &ocsp.RequestOptions{
+		Hash: crypto.SHA1,
+	})
+	if err != nil {
+		return nil, 0, errors.New("create ocsp request failed: " + err.Error())
+	}
+	ocspReq, err := http.NewRequest(http.MethodPost, ocspServerURL, bytes.NewBuffer(buf))
+	if err != nil {
+		return nil, 0, errors.New("request ocsp failed: " + err.Error())
+	}
+	ocspReq.Header.Set("Content-Type", "application/ocsp-request")
+	ocspReq.Header.Set("Accept", "application/ocsp-response")
+
+	ocspResp, err := this.httpClient.Do(ocspReq)
+	if err != nil {
+		return nil, 0, errors.New("request ocsp failed: '" + ocspServerURL + "': " + err.Error())
+	}
+
+	defer func() {
+		_ = ocspResp.Body.Close()
+	}()
+
+	respData, err := ioutil.ReadAll(ocspResp.Body)
+	if err != nil {
+		return nil, 0, errors.New("read ocsp failed: '" + ocspServerURL + "': " + err.Error())
+	}
+
+	ocspResult, err := ocsp.ParseResponse(respData, issuerCert)
+	if err != nil {
+		return nil, 0, errors.New("decode ocsp failed: " + err.Error())
+	}
+
+	// 只返回Good的ocsp
+	if ocspResult.Status == ocsp.Good {
+		return respData, ocspResult.NextUpdate.Unix(), nil
+	}
+	return nil, 0, nil
+}

internal/tasks/ssl_cert_update_ocsp_task_test.go

@@ -0,0 +1,19 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+
+package tasks_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/tasks"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+)
+
+func TestSSLCertUpdateOCSPTask_Loop(t *testing.T) {
+	dbs.NotifyReady()
+
+	var task = tasks.NewSSLCertUpdateOCSPTask()
+	err := task.Loop(false)
+	if err != nil {
+		t.Fatal(err)
+	}
+}