Compare commits

...

41 Commits

Author SHA1 Message Date
刘祥超
35cb10fffe 自动升级一个SQL注入规则 2022-03-20 11:54:17 +08:00
刘祥超
7c84794ac4 GRPC通讯支持gzip压缩 2022-03-20 11:28:45 +08:00
刘祥超
5859f5df91 更新相关库 2022-03-20 10:50:34 +08:00
刘祥超
e370944233 OCSP支持过期时间 2022-03-18 20:21:24 +08:00
刘祥超
a23a2ed826 获取OCSP更新列表兼容MySQL 5.7.22以下版本 2022-03-18 18:28:28 +08:00
刘祥超
3f9c250dff 动态更新OCSP,防止过期 2022-03-18 17:08:51 +08:00
刘祥超
06c9c9403b WAF统计图表中把page归为拦截,把tag归为记录 2022-03-17 16:49:46 +08:00
刘祥超
0e580a0890 源站支持自定义回源主机名 2022-03-17 15:48:40 +08:00
刘祥超
7aba622403 增加置顶集群功能 2022-03-17 11:12:46 +08:00
刘祥超
72cc8389e6 修复服务配置可能无法更新的Bug 2022-03-16 21:42:56 +08:00
刘祥超
39cf470b0c 上传SQL 2022-03-16 17:15:54 +08:00
刘祥超
6e71dda713 节点可以单独设置缓存目录 2022-03-16 15:23:58 +08:00
刘祥超
10909e28c8 修复集群配置/节点配置无法同步的Bug 2022-03-16 15:23:36 +08:00
刘祥超
6bd6e350b9 WAF全局看板中拦截类型只显示当天的统计/合并同名规则分组统计 2022-03-14 16:44:56 +08:00
刘祥超
d2ae7834ae 增加SQL子版本 2022-03-14 16:06:25 +08:00
刘祥超
c1ffd25c8b 上传SQL 2022-03-14 15:43:24 +08:00
刘祥超
931e55162b 缓存策略可以使用存储类型筛选 2022-03-14 15:42:48 +08:00
刘祥超
fa2bac6d1d 实现源站跟随功能 2022-03-14 15:42:45 +08:00
刘祥超
2f7b7240dd 增加证书OCSP错误日志管理 2022-03-11 20:27:53 +08:00
刘祥超
da67e726a2 优化代码 2022-03-11 17:39:08 +08:00
刘祥超
6cb5529c3f 证书ocsp获取失败时,报告错误 2022-03-11 13:02:53 +08:00
刘祥超
473d0d9439 修改OCSP自动更新的间隔时间和条数 2022-03-11 10:00:53 +08:00
刘祥超
bf0db231fc 上传sql 2022-03-10 21:33:08 +08:00
刘祥超
5644906b77 HTTP DNS QueryRecord动作支持返回null 2022-03-10 19:45:03 +08:00
刘祥超
3e32fe8e10 HTTP DNS请求时增加Content-Type 2022-03-10 19:13:53 +08:00
刘祥超
8459f106e9 域名操作错误时显示具体的域名、记录信息等 2022-03-10 17:42:23 +08:00
刘祥超
b768bbce5d 修改用户平台版本为0.3.2 2022-03-10 16:05:21 +08:00
刘祥超
9e7beb39c0 修复用户可能无法添加黑白名单IP的Bug 2022-03-10 15:59:00 +08:00
刘祥超
94287f5857 增加OCSP Stapling功能 2022-03-10 11:54:35 +08:00
刘祥超
60690dfd01 使用IPv6的IP搜索访问日志时自动去掉[] 2022-03-09 11:10:56 +08:00
刘祥超
fb00a7931e 支持使用小时筛选访问日志 2022-03-09 11:00:54 +08:00
刘祥超
42a6494bde 增加对访问日志自动分表配置 2022-03-09 10:01:24 +08:00
刘祥超
85a46a9827 自动对访问日志进行分表 2022-03-08 19:55:39 +08:00
刘祥超
088636553c 修复审计日志无法自动清理的Bug 2022-03-08 09:53:10 +08:00
刘祥超
95de3b12e2 缓存一些看板数据,以加快看板打开速度 2022-03-07 11:45:58 +08:00
刘祥超
b66b8d198a 更新相关库 2022-03-07 09:58:17 +08:00
刘祥超
e968a79886 更新TeaGo 2022-03-04 15:08:56 +08:00
刘祥超
4fc5d5b549 升级相关依赖库 2022-03-04 12:31:44 +08:00
刘祥超
77606709b3 增加是否同步写入压缩缓存设置 2022-02-24 20:11:53 +08:00
刘祥超
3529ceefcd 修改版本为0.4.5 2022-02-24 19:25:31 +08:00
刘祥超
7e851f07b1 修改版本为0.4.4 2022-02-23 14:49:12 +08:00
72 changed files with 2410 additions and 811 deletions

20
go.mod
View File

@@ -5,27 +5,23 @@ go 1.15
replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
require (
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183
github.com/andybalholm/brotli v1.0.4
github.com/cespare/xxhash/v2 v2.1.1
github.com/go-acme/lego/v4 v4.5.2
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/go-sql-driver/mysql v1.5.0
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/golang/protobuf v1.5.2
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
github.com/json-iterator/go v1.1.12 // indirect
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible
github.com/mozillazg/go-pinyin v0.18.0
github.com/pkg/sftp v1.12.0
github.com/shirou/gopsutil v3.21.5+incompatible
github.com/tklauser/go-sysconf v0.3.6 // indirect
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
google.golang.org/grpc v1.38.0
google.golang.org/protobuf v1.26.0
github.com/shirou/gopsutil/v3 v3.22.2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
google.golang.org/grpc v1.45.0
google.golang.org/protobuf v1.27.1
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

96
go.sum
View File

@@ -43,8 +43,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87/go.mod h1:iGLljf5n9GjT6kc0HBvyI1nOKnGQbNB66VzSNbK5iks=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -52,6 +50,7 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbx
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -78,6 +77,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cloudflare/cloudflare-go v0.20.0/go.mod h1:sPWL/lIC6biLEdyGZwBQ1rGQKF1FhM7N60fuNiFdYTI=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -101,7 +104,8 @@ github.com/dnsimple/dnsimple-go v0.70.1/go.mod h1:F9WHww9cC76hrnwGFfAfrqdW99j3MO
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/exoscale/egoscale v0.67.0/go.mod h1:wi0myUxPsV8SdEtdJHQJxFLL/wEw9fiw9Gs1PWRkvkM=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -122,8 +126,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
@@ -132,8 +136,6 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -161,6 +163,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -175,6 +178,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
@@ -201,6 +207,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
@@ -231,11 +238,8 @@ github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhK
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/infobloxopen/infoblox-go-client v1.1.1/go.mod h1:BXiw7S2b9qJoM8MS40vfgCNB2NLHGusk1DtO16BD9zI=
github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13 h1:HuEJ5xJfujW1Q6rNDhOu5LQXEBB2qLPah3jYslT8Gz4=
github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24 h1:1cGulkD2SNJJRok5OKwyhP/Ddm+PgSWKOupn0cR36/A=
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
@@ -250,8 +254,6 @@ github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -277,12 +279,11 @@ github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c
github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/linode/linodego v0.31.1/go.mod h1:BR0gVkCJffEdIGJSl6bHR80Ty+Uvg/2jkjmrWaFectM=
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible h1:1qp9iks+69h7IGLazAplzS9Ca14HAxuD5c0rbFdPGy4=
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible/go.mod h1:+ZBN7PBoh5gG6/y0ZQ85vJDBe21WnfbRrQQwTfliJJI=
github.com/liquidweb/go-lwApi v0.0.0-20190605172801-52a4864d2738/go.mod h1:0sYF9rMXb0vlG+4SzdiGMXHheCZxjguMq+Zb4S2BfBs=
github.com/liquidweb/go-lwApi v0.0.5/go.mod h1:0sYF9rMXb0vlG+4SzdiGMXHheCZxjguMq+Zb4S2BfBs=
github.com/liquidweb/liquidweb-cli v0.6.9/go.mod h1:cE1uvQ+x24NGUL75D0QagOFCG8Wdvmwu8aL9TLmA/eQ=
github.com/liquidweb/liquidweb-go v1.6.3/go.mod h1:SuXXp+thr28LnjEw18AYtWwIbWMHSUiajPQs8T9c/Rc=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -322,7 +323,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
@@ -371,6 +371,7 @@ github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
@@ -390,14 +391,17 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS7IBEjKVSrjg=
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -438,8 +442,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
@@ -456,13 +464,15 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -482,8 +492,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -507,7 +518,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -516,7 +526,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -551,8 +560,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -588,6 +599,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -611,6 +623,7 @@ golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -619,11 +632,17 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -631,8 +650,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -674,7 +694,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -714,9 +733,12 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
google.golang.org/genproto v0.0.0-20220303160752-862486edd9cc h1:fb/ViRpv3ln/LvbqZtTpoOd1YQDNH12gaGZreoSFovE=
google.golang.org/genproto v0.0.0-20220303160752-862486edd9cc/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -726,9 +748,12 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -739,8 +764,9 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@@ -2,8 +2,8 @@ package configs
import (
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"github.com/go-yaml/yaml"
"github.com/iwind/TeaGo/Tea"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
"path/filepath"

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.4.3"
Version = "0.4.5"
ProductName = "Edge API"
ProcessName = "edge-api"
@@ -18,10 +18,13 @@ const (
// 其他节点版本号,用来检测是否有需要升级的节点
NodeVersion = "0.4.3"
UserNodeVersion = "0.3.1"
NodeVersion = "0.4.5"
UserNodeVersion = "0.3.2"
AuthorityNodeVersion = "0.0.2"
MonitorNodeVersion = "0.0.3"
DNSNodeVersion = "0.2.1"
ReportNodeVersion = "0.1.0"
// SQLVersion SQL版本号
SQLVersion = "5"
)

View File

@@ -29,8 +29,7 @@ type httpAccessLogDefinition struct {
}
// HTTP服务访问
var httpAccessLogDAOMapping = map[int64]*HTTPAccessLogDAOWrapper{} // dbNodeId => DAO
var httpAccessLogTableMapping = map[string]*httpAccessLogDefinition{} // tableName_crc(dsn) => true
var httpAccessLogDAOMapping = map[int64]*HTTPAccessLogDAOWrapper{} // dbNodeId => DAO
// DNS服务访问
var nsAccessLogDAOMapping = map[int64]*NSAccessLogDAOWrapper{} // dbNodeId => DAO
@@ -86,36 +85,6 @@ func randomNSAccessLogDAO() (dao *NSAccessLogDAOWrapper) {
return
}
// 检查表格是否存在
func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, hasDomain bool, ok bool, err error) {
if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
return
}
config, err := db.Config()
if err != nil {
return "", false, false, false, err
}
tableName = "edgeHTTPAccessLogs_" + day
cacheKey := tableName + "_" + fmt.Sprintf("%d", crc32.ChecksumIEEE([]byte(config.Dsn)))
accessLogLocker.RLock()
def, ok := httpAccessLogTableMapping[cacheKey]
accessLogLocker.RUnlock()
if ok {
return tableName, def.HasRemoteAddr, def.HasDomain, true, nil
}
def, err = findHTTPAccessLogTable(db, day, false)
if err != nil {
return tableName, false, false, false, err
}
return tableName, def.HasRemoteAddr, def.HasDomain, def.Exists, nil
}
func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) {
if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
@@ -145,75 +114,6 @@ func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool
return tableName, utils.ContainsStringInsensitive(tableNames, tableName), nil
}
// 根据日期获取表名
func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
config, err := db.Config()
if err != nil {
return nil, err
}
tableName := "edgeHTTPAccessLogs_" + day
cacheKey := tableName + "_" + fmt.Sprintf("%d", crc32.ChecksumIEEE([]byte(config.Dsn)))
if !force {
accessLogLocker.RLock()
definition, ok := httpAccessLogTableMapping[cacheKey]
accessLogLocker.RUnlock()
if ok {
return definition, nil
}
}
tableNames, err := db.TableNames()
if err != nil {
return nil, err
}
if utils.ContainsStringInsensitive(tableNames, tableName) {
table, err := db.FindTable(tableName)
if err != nil {
return nil, err
}
accessLogLocker.Lock()
var definition = &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil,
HasDomain: table.FindFieldWithName("domain") != nil,
Exists: true,
}
httpAccessLogTableMapping[cacheKey] = definition
accessLogLocker.Unlock()
return definition, nil
}
if !force {
return &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: true,
HasDomain: true,
Exists: false,
}, nil
}
// 创建表格
_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',\n `domain` varchar(128) DEFAULT NULL COMMENT '域名',\n `requestBody` mediumblob COMMENT '请求内容',\n `responseBody` mediumblob COMMENT '响应内容',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`),\n KEY `remoteAddr` (`remoteAddr`),\n KEY `domain` (`domain`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
if err != nil {
return nil, err
}
accessLogLocker.Lock()
var definition = &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: true,
Exists: true,
}
httpAccessLogTableMapping[cacheKey] = definition
accessLogLocker.Unlock()
return definition, nil
}
func findNSAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
config, err := db.Config()
if err != nil {
@@ -351,22 +251,17 @@ func (this *DBNodeInitializer) loop() error {
// 检查表是否存在
// httpAccessLog
{
tableDef, err := findHTTPAccessLogTable(db, timeutil.Format("Ymd"), true)
tableDef, err := SharedHTTPAccessLogManager.FindTable(db, timeutil.Format("Ymd"), true)
if err != nil {
if !strings.Contains(err.Error(), "1050") { // 非表格已存在错误
remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
// 创建节点日志
createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix(), "", nil)
if createLogErr != nil {
remotelogs.Error("NODE_LOG", createLogErr.Error())
}
continue
} else {
err = nil
continue
// 创建节点日志
createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix(), "", nil)
if createLogErr != nil {
remotelogs.Error("NODE_LOG", createLogErr.Error())
}
continue
}
daoObject := dbs.DAOObject{

View File

@@ -1,9 +1,7 @@
package models
import (
"runtime"
"testing"
"time"
)
func TestDBNodeInitializer_loop(t *testing.T) {
@@ -14,32 +12,3 @@ func TestDBNodeInitializer_loop(t *testing.T) {
}
t.Log(len(accessLogDBMapping), len(httpAccessLogDAOMapping))
}
func TestFindAccessLogTable(t *testing.T) {
before := time.Now()
db := SharedHTTPAccessLogDAO.Instance
tableName, err := findHTTPAccessLogTable(db, "20201010", false)
if err != nil {
t.Fatal(err)
}
t.Log(tableName)
t.Log(time.Since(before).Seconds()*1000, "ms")
before = time.Now()
tableName, err = findHTTPAccessLogTable(db, "20201010", false)
if err != nil {
t.Fatal(err)
}
t.Log(tableName)
t.Log(time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkFindAccessLogTable(b *testing.B) {
db := SharedHTTPAccessLogDAO.Instance
runtime.GOMAXPROCS(1)
for i := 0; i < b.N; i++ {
_, _ = findHTTPAccessLogTable(db, "20201010", false)
}
}

View File

@@ -34,13 +34,25 @@ type HTTPAccessLogDAO dbs.DAO
var SharedHTTPAccessLogDAO *HTTPAccessLogDAO
// 队列
var oldAccessLogQueue = make(chan *pb.HTTPAccessLog)
var accessLogQueue = make(chan *pb.HTTPAccessLog, 10_000)
var accessLogQueueMaxLength = 100_000
var accessLogQueuePercent = 100 // 0-100
var accessLogCountPerSecond = 10_000 // 0 表示不限制
var accessLogConfigJSON = []byte{}
var accessLogQueueChanged = make(chan zero.Zero, 1)
var (
oldAccessLogQueue = make(chan *pb.HTTPAccessLog)
accessLogQueue = make(chan *pb.HTTPAccessLog, 10_000)
accessLogQueueMaxLength = 100_000
accessLogQueuePercent = 100 // 0-100
accessLogCountPerSecond = 10_000 // 0 表示不限制
accessLogConfigJSON = []byte{}
accessLogQueueChanged = make(chan zero.Zero, 1)
accessLogEnableAutoPartial = true // 是否启用自动分表
accessLogRowsPerTable int64 = 500_000 // 自动分表的单表最大值
)
type accessLogTableQuery struct {
daoWrapper *HTTPAccessLogDAOWrapper
name string
hasRemoteAddrField bool
hasDomainField bool
}
func init() {
dbs.OnReady(func() {
@@ -120,7 +132,7 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogs(tx *dbs.Tx, accessLogs []*pb.
// DumpAccessLogsFromQueue 从队列导入访问日志
func (this *HTTPAccessLogDAO) DumpAccessLogsFromQueue(tx *dbs.Tx, size int) error {
dao := randomHTTPAccessLogDAO()
var dao = randomHTTPAccessLogDAO()
if dao == nil {
dao = &HTTPAccessLogDAOWrapper{
DAO: SharedHTTPAccessLogDAO,
@@ -168,8 +180,8 @@ Loop:
// CreateHTTPAccessLog 写入单条访问日志
func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLogDAO, accessLog *pb.HTTPAccessLog) error {
day := timeutil.Format("Ymd", time.Unix(accessLog.Timestamp, 0))
tableDef, err := findHTTPAccessLogTable(dao.Instance, day, false)
var day = timeutil.Format("Ymd", time.Unix(accessLog.Timestamp, 0))
tableDef, err := SharedHTTPAccessLogManager.FindTable(dao.Instance, day, true)
if err != nil {
return err
}
@@ -203,27 +215,17 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLog
}
fields["content"] = content
_, err = dao.Query(tx).
var lastId int64
lastId, err = dao.Query(tx).
Table(tableDef.Name).
Sets(fields).
Insert()
if err != nil {
// 是否为 Error 1146: Table 'xxx.xxx' doesn't exist 如果是,则创建表之后重试
if strings.Contains(err.Error(), "1146") {
tableDef, err = findHTTPAccessLogTable(dao.Instance, day, true)
if err != nil {
return err
}
_, err = dao.Query(tx).
Table(tableDef.Name).
Sets(fields).
Insert()
if err != nil {
return err
}
} else {
remotelogs.Error("HTTP_ACCESS_LOG", err.Error())
}
return err
}
if accessLogEnableAutoPartial && accessLogRowsPerTable > 0 && lastId%accessLogRowsPerTable == 0 {
SharedHTTPAccessLogManager.ResetTable(dao.Instance, day)
}
return nil
@@ -233,6 +235,8 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLog
func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
size int64,
day string,
hourFrom string,
hourTo string,
clusterId int64,
nodeId int64,
serverId int64,
@@ -255,18 +259,36 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
size = 1000
}
result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
if err != nil || int64(len(result)) < size {
return
}
moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
hasMore = len(moreResult) > 0
return
}
// 读取往前的单页访问日志
func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, clusterId int64, nodeId int64, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string, ip string, domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
lastRequestId string,
size int64,
day string,
hourFrom string,
hourTo string,
clusterId int64,
nodeId int64,
serverId int64,
reverse bool,
hasError bool,
firewallPolicyId int64,
firewallRuleGroupId int64,
firewallRuleSetId int64,
hasFirewallPolicy bool,
userId int64,
keyword string,
ip string,
domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
if size <= 0 {
return nil, lastRequestId, nil
}
@@ -296,42 +318,57 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
}}
}
locker := sync.Mutex{}
// 查询某个集群下的节点
var nodeIds = []int64{}
if clusterId > 0 {
nodeIds, err = SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
if err != nil {
remotelogs.Error("DBNODE", err.Error())
return
}
sort.Slice(nodeIds, func(i, j int) bool {
return nodeIds[i] < nodeIds[j]
})
}
count := len(daoList)
wg := &sync.WaitGroup{}
wg.Add(count)
// 准备查询
var tableQueries = []*accessLogTableQuery{}
for _, daoWrapper := range daoList {
go func(daoWrapper *HTTPAccessLogDAOWrapper) {
var instance = daoWrapper.DAO.Instance
tableDefs, err := SharedHTTPAccessLogManager.FindTables(instance, day)
if err != nil {
return nil, "", err
}
for _, def := range tableDefs {
tableQueries = append(tableQueries, &accessLogTableQuery{
daoWrapper: daoWrapper,
name: def.Name,
hasRemoteAddrField: def.HasRemoteAddr,
hasDomainField: def.HasDomain,
})
}
}
var locker = sync.Mutex{}
var statusPrefixReg = regexp.MustCompile(`status:\s*(\d{3})\b`)
var statusRangeReg = regexp.MustCompile(`status:\s*(\d{3})-(\d{3})\b`)
var count = len(tableQueries)
var wg = &sync.WaitGroup{}
wg.Add(count)
for _, tableQuery := range tableQueries {
go func(tableQuery *accessLogTableQuery) {
defer wg.Done()
dao := daoWrapper.DAO
tableName, hasRemoteAddrField, hasDomainField, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
if err != nil {
logs.Println("[DB_NODE]" + err.Error())
return
}
if !exists {
// 表格不存在则跳过
return
}
query := dao.Query(tx)
var dao = tableQuery.daoWrapper.DAO
var query = dao.Query(tx)
// 条件
if nodeId > 0 {
query.Attr("nodeId", nodeId)
} else if clusterId > 0 {
nodeIds, err := SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
if err != nil {
remotelogs.Error("DBNODE", err.Error())
return
}
if len(nodeIds) > 0 {
sort.Slice(nodeIds, func(i, j int) bool {
return nodeIds[i] < nodeIds[j]
})
var nodeIdStrings = []string{}
for _, subNodeId := range nodeIds {
nodeIdStrings = append(nodeIdStrings, types.String(subNodeId))
@@ -370,7 +407,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
// keyword
if len(ip) > 0 {
// TODO 支持IP范围
if hasRemoteAddrField {
if tableQuery.hasRemoteAddrField {
// IP格式
if strings.Contains(ip, ",") || strings.Contains(ip, "-") {
rangeConfig, err := shared.ParseIPRange(ip)
@@ -380,6 +417,9 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
}
}
} else {
// 去掉IPv6的[]
ip = strings.Trim(ip, "[]")
query.Attr("remoteAddr", ip)
query.UseIndex("remoteAddr")
}
@@ -389,7 +429,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
}
}
if len(domain) > 0 {
if hasDomainField {
if tableQuery.hasDomainField {
if strings.Contains(domain, "*") {
domain = strings.ReplaceAll(domain, "*", "%")
domain = regexp.MustCompile(`[^a-zA-Z0-9-.%]`).ReplaceAllString(domain, "")
@@ -404,11 +444,12 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
Param("host1", domain)
}
}
if len(keyword) > 0 {
// remoteAddr
if hasRemoteAddrField && net.ParseIP(keyword) != nil {
if tableQuery.hasRemoteAddrField && net.ParseIP(keyword) != nil {
query.Attr("remoteAddr", keyword)
} else if hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
} else if tableQuery.hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
keyword = keyword[3:]
pieces := strings.SplitN(keyword, ",", 2)
if len(pieces) == 1 || len(pieces[1]) == 0 {
@@ -416,12 +457,21 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
} else {
query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
}
} else if statusRangeReg.MatchString(keyword) {
var matches = statusRangeReg.FindStringSubmatch(keyword)
query.Between("status", types.Int(matches[1]), types.Int(matches[2]))
// TODO 处理剩余的关键词
} else if statusPrefixReg.MatchString(keyword) {
var matches = statusPrefixReg.FindStringSubmatch(keyword)
query.Attr("status", matches[1])
// TODO 处理剩余的关键词
} else {
if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
keyword = keyword[3:]
}
useOriginKeyword := false
var useOriginKeyword = false
where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword OR JSON_EXTRACT(content, '$.userAgent') LIKE :keyword"
@@ -447,13 +497,13 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
// 响应状态码
if regexp.MustCompile(`^\d{3}$`).MatchString(keyword) {
where += " OR JSON_EXTRACT(content, '$.status')=:intKeyword"
where += " OR status=:intKeyword"
query.Param("intKeyword", types.Int(keyword))
}
if regexp.MustCompile(`^\d{3}-\d{3}$`).MatchString(keyword) {
pieces := strings.Split(keyword, "-")
where += " OR JSON_EXTRACT(content, '$.status') BETWEEN :intKeyword1 AND :intKeyword2"
where += " OR status BETWEEN :intKeyword1 AND :intKeyword2"
query.Param("intKeyword1", types.Int(pieces[0]))
query.Param("intKeyword2", types.Int(pieces[1]))
}
@@ -471,6 +521,20 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
}
}
// hourFrom - hourTo
if len(hourFrom) > 0 && len(hourTo) > 0 {
var hourFromInt = types.Int(hourFrom)
var hourToInt = types.Int(hourTo)
if hourFromInt >= 0 && hourFromInt <= 23 && hourToInt >= hourFromInt && hourToInt <= 23 {
var y = types.Int(day[:4])
var m = types.Int(day[4:6])
var d = types.Int(day[6:])
var timeFrom = time.Date(y, time.Month(m), d, hourFromInt, 0, 0, 0, time.Local)
var timeTo = time.Date(y, time.Month(m), d, hourToInt, 59, 59, 0, time.Local)
query.Between("createdAt", timeFrom.Unix(), timeTo.Unix())
}
}
// offset
if len(lastRequestId) > 0 {
if !reverse {
@@ -490,20 +554,21 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
// 开始查询
ones, err := query.
Table(tableName).
Table(tableQuery.name).
Limit(size).
FindAll()
if err != nil {
logs.Println("[DB_NODE]" + err.Error())
return
}
locker.Lock()
for _, one := range ones {
accessLog := one.(*HTTPAccessLog)
result = append(result, accessLog)
}
locker.Unlock()
}(daoWrapper)
}(tableQuery)
}
wg.Wait()
@@ -524,7 +589,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
result = result[:size]
}
requestId := result[len(result)-1].RequestId
var requestId = result[len(result)-1].RequestId
if reverse {
lists.Reverse(result)
}
@@ -556,28 +621,36 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s
}}
}
count := len(daoList)
wg := &sync.WaitGroup{}
// 准备查询
var day = timeutil.FormatTime("Ymd", types.Int64(requestId[:10]))
var tableQueries = []*accessLogTableQuery{}
for _, daoWrapper := range daoList {
var instance = daoWrapper.DAO.Instance
tableDefs, err := SharedHTTPAccessLogManager.FindTables(instance, day)
if err != nil {
return nil, err
}
for _, def := range tableDefs {
tableQueries = append(tableQueries, &accessLogTableQuery{
daoWrapper: daoWrapper,
name: def.Name,
hasRemoteAddrField: def.HasRemoteAddr,
hasDomainField: def.HasDomain,
})
}
}
var count = len(tableQueries)
var wg = &sync.WaitGroup{}
wg.Add(count)
var result *HTTPAccessLog = nil
day := timeutil.FormatTime("Ymd", types.Int64(requestId[:10]))
for _, daoWrapper := range daoList {
go func(daoWrapper *HTTPAccessLogDAOWrapper) {
for _, tableQuery := range tableQueries {
go func(tableQuery *accessLogTableQuery) {
defer wg.Done()
dao := daoWrapper.DAO
tableName, _, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
if err != nil {
logs.Println("[DB_NODE]" + err.Error())
return
}
if !exists {
return
}
var dao = tableQuery.daoWrapper.DAO
one, err := dao.Query(tx).
Table(tableName).
Table(tableQuery.name).
Attr("requestId", requestId).
Find()
if err != nil {
@@ -587,7 +660,7 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s
if one != nil {
result = one.(*HTTPAccessLog)
}
}(daoWrapper)
}(tableQuery)
}
wg.Wait()
return result, nil
@@ -623,11 +696,18 @@ func (this *HTTPAccessLogDAO) SetupQueue() {
config.MaxLength = 100_000
}
accessLogEnableAutoPartial = config.EnableAutoPartial
if config.RowsPerTable > 0 {
accessLogRowsPerTable = config.RowsPerTable
}
if accessLogQueueMaxLength != config.MaxLength {
accessLogQueueMaxLength = config.MaxLength
oldAccessLogQueue = accessLogQueue
accessLogQueue = make(chan *pb.HTTPAccessLog, config.MaxLength)
}
remotelogs.Println("HTTP_ACCESS_LOG_QUEUE", "change queue max length: "+types.String(config.MaxLength)+", percent: "+types.String(config.Percent)+", countPerSecond: "+types.String(config.CountPerSecond))
if Tea.IsTesting() {
remotelogs.Println("HTTP_ACCESS_LOG_QUEUE", "change queue "+string(configJSON))
}
}

View File

@@ -53,7 +53,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
t.Fatal(err)
}
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
if err != nil {
t.Fatal(err)
}
@@ -80,7 +80,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
times := 0 // 防止循环次数太多
for {
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)
@@ -111,7 +111,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
}
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)
@@ -136,7 +136,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
times := 0 // 防止循环次数太多
for {
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)

View File

@@ -0,0 +1,297 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package models
import (
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"regexp"
"sort"
"strings"
"sync"
)
// 访问日志的两个表格形式
var accessLogTableMainReg = regexp.MustCompile(`_(\d{8})$`)
var accessLogTablePartialReg = regexp.MustCompile(`_(\d{8})_(\d{4})$`)
var SharedHTTPAccessLogManager = NewHTTPAccessLogManager()
type HTTPAccessLogManager struct {
currentTableMapping map[string]*httpAccessLogDefinition // dsn => def
locker sync.Mutex
}
func NewHTTPAccessLogManager() *HTTPAccessLogManager {
return &HTTPAccessLogManager{
currentTableMapping: map[string]*httpAccessLogDefinition{},
}
}
// FindTableNames 读取数据库中某日所有日志表名称
func (this *HTTPAccessLogManager) FindTableNames(db *dbs.DB, day string) ([]string, error) {
var results = []string{}
// 需要防止用户设置了表名自动小写
for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
if err != nil {
return nil, errors.New("query table names error: " + err.Error())
}
var columnName = columnNames[0]
for _, one := range ones {
var tableName = one[columnName].(string)
if lists.ContainsString(results, tableName) {
continue
}
if accessLogTableMainReg.MatchString(tableName) || accessLogTablePartialReg.MatchString(tableName) {
results = append(results, tableName)
}
}
}
// 排序
sort.Strings(results)
return results, nil
}
// FindTables 读取数据库中某日所有日志表
func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAccessLogDefinition, error) {
var results = []*httpAccessLogDefinition{}
var tableNames = []string{}
// 需要防止用户设置了表名自动小写
for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
if err != nil {
return nil, errors.New("query table names error: " + err.Error())
}
var columnName = columnNames[0]
for _, one := range ones {
var tableName = one[columnName].(string)
if lists.ContainsString(tableNames, tableName) {
continue
}
if accessLogTableMainReg.MatchString(tableName) {
tableNames = append(tableNames, tableName)
hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, tableName)
if err != nil {
return nil, err
}
results = append(results, &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: hasRemoteAddrField,
HasDomain: hasDomainField,
Exists: true,
})
} else if accessLogTablePartialReg.MatchString(tableName) {
tableNames = append(tableNames, tableName)
results = append(results, &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: true,
HasDomain: true,
Exists: true,
})
}
}
}
// 排序
sort.Slice(results, func(i, j int) bool {
return results[i].Name < results[j].Name
})
return results, nil
}
// FindTable 根据日期获取表名
// 表名组成
// - PREFIX_DAY
// - PREFIX_DAY_0001
func (this *HTTPAccessLogManager) FindTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
this.locker.Lock()
defer this.locker.Unlock()
config, err := db.Config()
if err != nil {
return nil, err
}
var cacheKey = config.Dsn
def, ok := this.currentTableMapping[cacheKey]
if ok {
return def, nil
}
def, err = this.findTableWithoutCache(db, day, force)
if err != nil {
return nil, err
}
this.currentTableMapping[cacheKey] = def
return def, nil
}
// CreateTable 创建访问日志表格
func (this *HTTPAccessLogManager) CreateTable(db *dbs.DB, tableName string) error {
_, err := db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',\n `domain` varchar(128) DEFAULT NULL COMMENT '域名',\n `requestBody` mediumblob COMMENT '请求内容',\n `responseBody` mediumblob COMMENT '响应内容',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`),\n KEY `remoteAddr` (`remoteAddr`),\n KEY `domain` (`domain`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
if err != nil {
// 快速判断错误方法
mysqlErr, ok := err.(*mysql.MySQLError)
if ok && mysqlErr.Number == 1050 { // Error 1050: Table 'xxx' already exists
return nil
}
// 防止二次包装过程中错误丢失的保底错误判断方法
if strings.Contains(err.Error(), "Error 1050") {
return nil
}
return err
}
return nil
}
// ResetTable 清除某个数据库表名缓存
func (this *HTTPAccessLogManager) ResetTable(db *dbs.DB, day string) {
this.locker.Lock()
defer this.locker.Unlock()
config, err := db.Config()
if err != nil {
return
}
delete(this.currentTableMapping, config.Dsn)
}
// 查找某个表格
func (this *HTTPAccessLogManager) findTableWithoutCache(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
tableNames, err := this.FindTableNames(db, day)
if err != nil {
return nil, err
}
var prefix = "edgeHTTPAccessLogs_" + day
if len(tableNames) == 0 {
if force {
err := this.CreateTable(db, prefix)
if err != nil {
return nil, err
}
return &httpAccessLogDefinition{
Name: prefix,
HasRemoteAddr: true,
HasDomain: true,
Exists: true,
}, nil
}
return &httpAccessLogDefinition{
Name: prefix,
HasRemoteAddr: true,
HasDomain: true,
Exists: false,
}, nil
}
var lastTableName = tableNames[len(tableNames)-1]
if !force || !accessLogEnableAutoPartial || accessLogRowsPerTable <= 0 {
hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, lastTableName)
if err != nil {
return nil, err
}
return &httpAccessLogDefinition{
Name: lastTableName,
HasRemoteAddr: hasRemoteAddrField,
HasDomain: hasDomainField,
Exists: true,
}, nil
}
// 检查是否生成下个分表
lastId, err := db.FindCol(0, "SELECT id FROM "+lastTableName+" ORDER BY id DESC LIMIT 1")
if err != nil {
return nil, err
}
if lastId != nil {
var lastInt64Id = types.Int64(lastId)
if accessLogRowsPerTable > 0 && lastInt64Id >= accessLogRowsPerTable {
// create next partial table
var nextTableName = ""
if accessLogTableMainReg.MatchString(lastTableName) {
nextTableName = prefix + "_0001"
} else if accessLogTablePartialReg.MatchString(lastTableName) {
var matches = accessLogTablePartialReg.FindStringSubmatch(lastTableName)
if len(matches) < 3 {
return nil, errors.New("fatal error: invalid 'accessLogTablePartialReg'")
}
var lastPartial = matches[2]
nextTableName = prefix + "_" + fmt.Sprintf("%04d", types.Int(lastPartial)+1)
} else {
nextTableName = prefix + "_0001"
}
err = this.CreateTable(db, nextTableName)
if err != nil {
return nil, err
}
return &httpAccessLogDefinition{
Name: nextTableName,
HasRemoteAddr: true,
HasDomain: true,
Exists: true,
}, nil
}
}
// 检查字段
hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, lastTableName)
if err != nil {
return nil, err
}
return &httpAccessLogDefinition{
Name: lastTableName,
HasRemoteAddr: hasRemoteAddrField,
HasDomain: hasDomainField,
Exists: true,
}, nil
}
// TODO 考虑缓存检查结果
func (this *HTTPAccessLogManager) checkTableFields(db *dbs.DB, tableName string) (hasRemoteAddrField bool, hasDomainField bool, err error) {
fields, _, err := db.FindOnes("SHOW FIELDS FROM " + tableName)
if err != nil {
return false, false, err
}
for _, field := range fields {
var fieldName = field.GetString("Field")
if strings.ToLower(fieldName) == strings.ToLower("remoteAddr") {
hasRemoteAddrField = true
}
if strings.ToLower(fieldName) == "domain" {
hasDomainField = true
}
}
return
}

View File

@@ -0,0 +1,148 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package models_test
import (
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/iwind/TeaGo/dbs"
"testing"
"time"
)
func TestNewHTTPAccessLogManager(t *testing.T) {
var config = &dbs.DBConfig{
Driver: "mysql",
Dsn: "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
Prefix: "edge",
Connections: struct {
Pool int `yaml:"pool"`
Max int `yaml:"max"`
Life string `yaml:"life"`
LifeDuration time.Duration `yaml:",omitempty"`
}{},
Models: struct {
Package string `yaml:"package"`
}{},
}
db, err := dbs.NewInstanceFromConfig(config)
if err != nil {
t.Fatal(err)
}
var manager = models.SharedHTTPAccessLogManager
err = manager.CreateTable(db, "accessLog_1")
if err != nil {
t.Fatal(err)
}
}
func TestHTTPAccessLogManager_FindTableNames(t *testing.T) {
var config = &dbs.DBConfig{
Driver: "mysql",
Dsn: "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
Prefix: "edge",
Connections: struct {
Pool int `yaml:"pool"`
Max int `yaml:"max"`
Life string `yaml:"life"`
LifeDuration time.Duration `yaml:",omitempty"`
}{},
Models: struct {
Package string `yaml:"package"`
}{},
}
db, err := dbs.NewInstanceFromConfig(config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
var before = time.Now()
tables, err := models.SharedHTTPAccessLogManager.FindTables(db, "20220306")
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(tables)
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
t.Log(time.Since(before).Seconds()*1000, "ms")
}
}
func TestHTTPAccessLogManager_FindTables(t *testing.T) {
var config = &dbs.DBConfig{
Driver: "mysql",
Dsn: "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
Prefix: "edge",
Connections: struct {
Pool int `yaml:"pool"`
Max int `yaml:"max"`
Life string `yaml:"life"`
LifeDuration time.Duration `yaml:",omitempty"`
}{},
Models: struct {
Package string `yaml:"package"`
}{},
}
db, err := dbs.NewInstanceFromConfig(config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
var before = time.Now()
tables, err := models.SharedHTTPAccessLogManager.FindTables(db, "20220306")
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(tables)
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
t.Log(time.Since(before).Seconds()*1000, "ms")
}
}
func TestHTTPAccessLogManager_FindTable(t *testing.T) {
var config = &dbs.DBConfig{
Driver: "mysql",
Dsn: "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
Prefix: "edge",
Connections: struct {
Pool int `yaml:"pool"`
Max int `yaml:"max"`
Life string `yaml:"life"`
LifeDuration time.Duration `yaml:",omitempty"`
}{},
Models: struct {
Package string `yaml:"package"`
}{},
}
db, err := dbs.NewInstanceFromConfig(config)
if err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
var before = time.Now()
tableDef, err := models.SharedHTTPAccessLogManager.FindTable(db, "20220306", false)
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(tableDef)
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
t.Log(time.Since(before).Seconds()*1000, "ms")
}
}

View File

@@ -95,7 +95,7 @@ func (this *HTTPCachePolicyDAO) FindAllEnabledCachePolicies(tx *dbs.Tx) (result
}
// CreateCachePolicy 创建缓存策略
func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte) (int64, error) {
func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) (int64, error) {
op := NewHTTPCachePolicyOperator()
op.State = HTTPCachePolicyStateEnabled
op.IsOn = isOn
@@ -112,6 +112,7 @@ func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name st
if len(storageOptionsJSON) > 0 {
op.Options = storageOptionsJSON
}
op.SyncCompressionCache = syncCompressionCache
// 默认的缓存条件
cacheRef := &serverconfigs.HTTPCacheRef{
@@ -182,13 +183,19 @@ func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string
var storageOptions = &serverconfigs.HTTPFileCacheStorage{
Dir: "/opt/cache",
MemoryPolicy: &serverconfigs.HTTPCachePolicy{
Capacity: &shared.SizeCapacity{
Count: 1,
Unit: shared.SizeCapacityUnitGB,
},
},
}
storageOptionsJSON, err := json.Marshal(storageOptions)
if err != nil {
return 0, err
}
policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, 0, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON)
policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, 0, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON, false)
if err != nil {
return 0, err
}
@@ -196,7 +203,7 @@ func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string
}
// UpdateCachePolicy 修改缓存策略
func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte) error {
func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -217,6 +224,7 @@ func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, is
if len(storageOptionsJSON) > 0 {
op.Options = storageOptionsJSON
}
op.SyncCompressionCache = syncCompressionCache
err := this.Save(tx, op)
if err != nil {
return err
@@ -247,6 +255,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
config.IsOn = policy.IsOn == 1
config.Name = policy.Name
config.Description = policy.Description
config.SyncCompressionCache = policy.SyncCompressionCache == 1
// capacity
if IsNotNull(policy.Capacity) {
@@ -300,7 +309,7 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, c
}
// CountAllEnabledHTTPCachePolicies 计算可用缓存策略数量
func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string) (int64, error) {
func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string, storageType string) (int64, error) {
query := this.Query(tx).
State(HTTPCachePolicyStateEnabled)
if clusterId > 0 {
@@ -311,11 +320,14 @@ func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clu
query.Where("(name LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
}
if len(storageType) > 0 {
query.Attr("type", storageType)
}
return query.Count()
}
// ListEnabledHTTPCachePolicies 列出单页的缓存策略
func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string, offset int64, size int64) ([]*serverconfigs.HTTPCachePolicy, error) {
func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, clusterId int64, keyword string, storageType string, offset int64, size int64) ([]*serverconfigs.HTTPCachePolicy, error) {
query := this.Query(tx).
State(HTTPCachePolicyStateEnabled)
if clusterId > 0 {
@@ -326,6 +338,9 @@ func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, cluster
query.Where("(name LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
}
if len(storageType) > 0 {
query.Attr("type", storageType)
}
ones, err := query.
ResultPk().
Offset(offset).

View File

@@ -2,39 +2,41 @@ package models
// HTTPCachePolicy HTTP缓存策略
type HTTPCachePolicy struct {
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID
TemplateId uint32 `field:"templateId"` // 模版ID
IsOn uint8 `field:"isOn"` // 是否启用
Name string `field:"name"` // 名称
Capacity string `field:"capacity"` // 容量数据
MaxKeys uint64 `field:"maxKeys"` // 最多Key值
MaxSize string `field:"maxSize"` // 最大缓存内容尺寸
Type string `field:"type"` // 存储类型
Options string `field:"options"` // 存储选项
CreatedAt uint64 `field:"createdAt"` // 创建时间
State uint8 `field:"state"` // 状态
Description string `field:"description"` // 描述
Refs string `field:"refs"` // 默认的缓存设置
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID
TemplateId uint32 `field:"templateId"` // 模版ID
IsOn uint8 `field:"isOn"` // 是否启用
Name string `field:"name"` // 名称
Capacity string `field:"capacity"` // 容量数据
MaxKeys uint64 `field:"maxKeys"` // 最多Key值
MaxSize string `field:"maxSize"` // 最大缓存内容尺寸
Type string `field:"type"` // 存储类型
Options string `field:"options"` // 存储选项
CreatedAt uint64 `field:"createdAt"` // 创建时间
State uint8 `field:"state"` // 状态
Description string `field:"description"` // 描述
Refs string `field:"refs"` // 默认的缓存设置
SyncCompressionCache uint8 `field:"syncCompressionCache"` // 是否同步写入压缩缓存
}
type HTTPCachePolicyOperator struct {
Id interface{} // ID
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
TemplateId interface{} // 模版ID
IsOn interface{} // 是否启用
Name interface{} // 名称
Capacity interface{} // 容量数据
MaxKeys interface{} // 最多Key值
MaxSize interface{} // 最大缓存内容尺寸
Type interface{} // 存储类型
Options interface{} // 存储选项
CreatedAt interface{} // 创建时间
State interface{} // 状态
Description interface{} // 描述
Refs interface{} // 默认的缓存设置
Id interface{} // ID
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
TemplateId interface{} // 模版ID
IsOn interface{} // 是否启用
Name interface{} // 名称
Capacity interface{} // 容量数据
MaxKeys interface{} // 最多Key值
MaxSize interface{} // 最大缓存内容尺寸
Type interface{} // 存储类型
Options interface{} // 存储选项
CreatedAt interface{} // 创建时间
State interface{} // 状态
Description interface{} // 描述
Refs interface{} // 默认的缓存设置
SyncCompressionCache interface{} // 是否同步写入压缩缓存
}
func NewHTTPCachePolicyOperator() *HTTPCachePolicyOperator {

View File

@@ -184,6 +184,10 @@ func (this *IPListDAO) IncreaseVersion(tx *dbs.Tx) (int64, error) {
// CheckUserIPList 检查用户权限
func (this *IPListDAO) CheckUserIPList(tx *dbs.Tx, userId int64, listId int64) error {
if userId == 0 || listId == 0 {
return ErrNotFound
}
ok, err := this.Query(tx).
Pk(listId).
Attr("userId", userId).
@@ -194,6 +198,18 @@ func (this *IPListDAO) CheckUserIPList(tx *dbs.Tx, userId int64, listId int64) e
if ok {
return nil
}
// 检查是否被用户的服务所使用
policyIds, err := SharedHTTPFirewallPolicyDAO.FindEnabledFirewallPolicyIdsWithIPListId(tx, listId)
if err != nil {
return err
}
for _, policyId := range policyIds {
if SharedHTTPFirewallPolicyDAO.CheckUserFirewallPolicy(tx, userId, policyId) == nil {
return nil
}
}
return ErrNotFound
}

View File

@@ -33,7 +33,7 @@ func init() {
})
}
// 创建管理员日志
// CreateLog 创建管理员日志
func (this *LogDAO) CreateLog(tx *dbs.Tx, adminType string, adminId int64, level string, description string, action string, ip string) error {
op := NewLogOperator()
op.Level = level
@@ -57,7 +57,7 @@ func (this *LogDAO) CreateLog(tx *dbs.Tx, adminType string, adminId int64, level
return err
}
// 计算所有日志数量
// CountLogs 计算所有日志数量
func (this *LogDAO) CountLogs(tx *dbs.Tx, dayFrom string, dayTo string, keyword string, userType string) (int64, error) {
dayFrom = this.formatDay(dayFrom)
dayTo = this.formatDay(dayTo)
@@ -86,7 +86,7 @@ func (this *LogDAO) CountLogs(tx *dbs.Tx, dayFrom string, dayTo string, keyword
return query.Count()
}
// 列出单页日志
// ListLogs 列出单页日志
func (this *LogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64, dayFrom string, dayTo string, keyword string, userType string) (result []*Log, err error) {
dayFrom = this.formatDay(dayFrom)
dayTo = this.formatDay(dayTo)
@@ -120,7 +120,7 @@ func (this *LogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64, dayFrom strin
return
}
// 物理删除日志
// DeleteLogPermanently 物理删除日志
func (this *LogDAO) DeleteLogPermanently(tx *dbs.Tx, logId int64) error {
if logId <= 0 {
return errors.New("invalid logId")
@@ -129,14 +129,14 @@ func (this *LogDAO) DeleteLogPermanently(tx *dbs.Tx, logId int64) error {
return err
}
// 物理删除所有日志
// DeleteAllLogsPermanently 物理删除所有日志
func (this *LogDAO) DeleteAllLogsPermanently(tx *dbs.Tx) error {
_, err := this.Query(tx).
Delete()
return err
}
// 物理删除某些天之前的日志
// DeleteLogsPermanentlyBeforeDays 物理删除某些天之前的日志
func (this *LogDAO) DeleteLogsPermanentlyBeforeDays(tx *dbs.Tx, days int) error {
if days <= 0 {
days = 0
@@ -148,7 +148,7 @@ func (this *LogDAO) DeleteLogsPermanentlyBeforeDays(tx *dbs.Tx, days int) error
return err
}
// 计算当前日志容量大小
// SumLogsSize 计算当前日志容量大小
func (this *LogDAO) SumLogsSize() (int64, error) {
col, err := this.Instance.FindCol(0, "SELECT DATA_LENGTH FROM information_schema.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=? LIMIT 1", this.Instance.Name(), this.Table)
if err != nil {

View File

@@ -207,6 +207,14 @@ func (this *NodeClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name stri
return this.NotifyUpdate(tx, clusterId)
}
// UpdateClusterIsPinned 设置集群是否置顶
func (this *NodeClusterDAO) UpdateClusterIsPinned(tx *dbs.Tx, clusterId int64, isPinned bool) error {
return this.Query(tx).
Pk(clusterId).
Set("isPinned", isPinned).
UpdateQuickly()
}
// CountAllEnabledClusters 计算所有集群数量
func (this *NodeClusterDAO) CountAllEnabledClusters(tx *dbs.Tx, keyword string) (int64, error) {
query := this.Query(tx).
@@ -230,6 +238,7 @@ func (this *NodeClusterDAO) ListEnabledClusters(tx *dbs.Tx, keyword string, offs
Offset(offset).
Limit(size).
Slice(&result).
Desc("isPinned").
DescPk().
FindAll()
return

View File

@@ -30,6 +30,7 @@ type NodeCluster struct {
NodeMaxThreads uint32 `field:"nodeMaxThreads"` // 节点最大线程数
NodeTCPMaxConnections uint32 `field:"nodeTCPMaxConnections"` // TCP最大连接数
AutoOpenPorts uint8 `field:"autoOpenPorts"` // 是否自动尝试开放端口
IsPinned uint8 `field:"isPinned"` // 是否置顶
}
type NodeClusterOperator struct {
@@ -61,6 +62,7 @@ type NodeClusterOperator struct {
NodeMaxThreads interface{} // 节点最大线程数
NodeTCPMaxConnections interface{} // TCP最大连接数
AutoOpenPorts interface{} // 是否自动尝试开放端口
IsPinned interface{} // 是否置顶
}
func NewNodeClusterOperator() *NodeClusterOperator {

View File

@@ -456,9 +456,26 @@ func (this *NodeDAO) FindEnabledNodeClusterIds(tx *dbs.Tx, nodeId int64) (result
return
}
// FindEnabledNodeIdsWithClusterId 查找某个集群下的所有节点IDs
func (this *NodeDAO) FindEnabledNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64) ([]int64, error) {
ones, err := this.Query(tx).
Attr("clusterId", clusterId).
State(NodeClusterStateEnabled).
ResultPk().
FindAll()
if err != nil {
return nil, err
}
var result = []int64{}
for _, one := range ones {
result = append(result, int64(one.(*Node).Id))
}
return result, nil
}
// FindAllNodeIdsMatch 匹配节点并返回节点ID
func (this *NodeDAO) FindAllNodeIdsMatch(tx *dbs.Tx, clusterId int64, includeSecondaryNodes bool, isOn configutils.BoolState) (result []int64, err error) {
query := this.Query(tx)
var query = this.Query(tx)
query.State(NodeStateEnabled)
if clusterId > 0 {
if includeSecondaryNodes {
@@ -856,6 +873,8 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
}
}
config.CacheDiskDir = node.CacheDiskDir
// TOA
toaConfig, err := SharedNodeClusterDAO.FindClusterTOAConfig(tx, primaryClusterId, cacheMap)
if err != nil {
@@ -929,6 +948,13 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
}
}
// OCSP
ocspVersion, err := SharedSSLCertDAO.FindCertOCSPLatestVersion(tx)
if err != nil {
return nil, err
}
config.OCSPVersion = ocspVersion
return config, nil
}
@@ -1205,7 +1231,7 @@ func (this *NodeDAO) UpdateNodeSystem(tx *dbs.Tx, nodeId int64, maxCPU int32) er
}
// UpdateNodeCache 设置缓存相关
func (this *NodeDAO) UpdateNodeCache(tx *dbs.Tx, nodeId int64, maxCacheDiskCapacityJSON []byte, maxCacheMemoryCapacityJSON []byte) error {
func (this *NodeDAO) UpdateNodeCache(tx *dbs.Tx, nodeId int64, maxCacheDiskCapacityJSON []byte, maxCacheMemoryCapacityJSON []byte, cacheDiskDir string) error {
if nodeId <= 0 {
return errors.New("invalid nodeId")
}
@@ -1217,6 +1243,7 @@ func (this *NodeDAO) UpdateNodeCache(tx *dbs.Tx, nodeId int64, maxCacheDiskCapac
if len(maxCacheMemoryCapacityJSON) > 0 {
op.MaxCacheMemoryCapacity = maxCacheMemoryCapacityJSON
}
op.CacheDiskDir = cacheDiskDir
err := this.Save(tx, op)
if err != nil {
return err

View File

@@ -31,6 +31,7 @@ type Node struct {
DnsRoutes string `field:"dnsRoutes"` // DNS线路设置
MaxCacheDiskCapacity string `field:"maxCacheDiskCapacity"` // 硬盘缓存容量
MaxCacheMemoryCapacity string `field:"maxCacheMemoryCapacity"` // 内存缓存容量
CacheDiskDir string `field:"cacheDiskDir"` // 缓存目录
}
type NodeOperator struct {
@@ -63,6 +64,7 @@ type NodeOperator struct {
DnsRoutes interface{} // DNS线路设置
MaxCacheDiskCapacity interface{} // 硬盘缓存容量
MaxCacheMemoryCapacity interface{} // 内存缓存容量
CacheDiskDir interface{} // 缓存目录
}
func NewNodeOperator() *NodeOperator {

View File

@@ -53,8 +53,8 @@ func (this *NodeTaskDAO) CreateNodeTask(tx *dbs.Tx, role string, clusterId int64
if clusterId <= 0 || nodeId <= 0 {
return nil
}
uniqueId := role + "@" + types.String(nodeId) + "@node@" + taskType
updatedAt := time.Now().Unix()
var uniqueId = role + "@" + types.String(nodeId) + "@node@" + types.String(serverId) + "@" + taskType
var updatedAt = time.Now().Unix()
_, _, err := this.Query(tx).
InsertOrUpdate(maps.Map{
"role": role,
@@ -76,6 +76,7 @@ func (this *NodeTaskDAO) CreateNodeTask(tx *dbs.Tx, role string, clusterId int64
"error": "",
"isNotified": 0,
"version": version,
"serverId": serverId,
})
return err
}
@@ -86,8 +87,8 @@ func (this *NodeTaskDAO) CreateClusterTask(tx *dbs.Tx, role string, clusterId in
return nil
}
uniqueId := role + "@" + types.String(clusterId) + "@" + types.String(serverId) + "@cluster@" + taskType
updatedAt := time.Now().Unix()
var uniqueId = role + "@" + types.String(clusterId) + "@" + types.String(serverId) + "@cluster@" + types.String(serverId) + "@" + taskType
var updatedAt = time.Now().Unix()
_, _, err := this.Query(tx).
InsertOrUpdate(maps.Map{
"role": role,
@@ -109,6 +110,7 @@ func (this *NodeTaskDAO) CreateClusterTask(tx *dbs.Tx, role string, clusterId in
"isNotified": 0,
"error": "",
"version": time.Now().UnixNano(),
"serverId": serverId,
})
return err
}

View File

@@ -100,8 +100,9 @@ func (this *OriginDAO) CreateOrigin(tx *dbs.Tx,
maxConns int32,
maxIdleConns int32,
certRef *sslconfigs.SSLCertRef,
domains []string) (originId int64, err error) {
op := NewOriginOperator()
domains []string,
host string) (originId int64, err error) {
var op = NewOriginOperator()
op.AdminId = adminId
op.UserId = userId
op.IsOn = isOn
@@ -165,6 +166,8 @@ func (this *OriginDAO) CreateOrigin(tx *dbs.Tx,
op.Domains = "[]"
}
op.Host = host
op.State = OriginStateEnabled
err = this.Save(tx, op)
if err != nil {
@@ -187,11 +190,12 @@ func (this *OriginDAO) UpdateOrigin(tx *dbs.Tx,
maxConns int32,
maxIdleConns int32,
certRef *sslconfigs.SSLCertRef,
domains []string) error {
domains []string,
host string) error {
if originId <= 0 {
return errors.New("invalid originId")
}
op := NewOriginOperator()
var op = NewOriginOperator()
op.Id = originId
op.Name = name
op.Addr = addrJSON
@@ -257,6 +261,8 @@ func (this *OriginDAO) UpdateOrigin(tx *dbs.Tx,
op.Domains = "[]"
}
op.Host = host
err := this.Save(tx, op)
if err != nil {
return err
@@ -284,7 +290,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
return nil, nil
}
config := &serverconfigs.OriginConfig{
var config = &serverconfigs.OriginConfig{
Id: int64(origin.Id),
IsOn: origin.IsOn == 1,
Version: int(origin.Version),

View File

@@ -107,6 +107,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
config.RequestURI = reverseProxy.RequestURI
config.StripPrefix = reverseProxy.StripPrefix
config.AutoFlush = reverseProxy.AutoFlush == 1
config.FollowRedirects = reverseProxy.FollowRedirects == 1
schedulingConfig := &serverconfigs.SchedulingConfig{}
if len(reverseProxy.Scheduling) > 0 && reverseProxy.Scheduling != "null" {
@@ -312,12 +313,13 @@ func (this *ReverseProxyDAO) UpdateReverseProxy(tx *dbs.Tx,
idleTimeout *shared.TimeDuration,
maxConns int32,
maxIdleConns int32,
proxyProtocolJSON []byte) error {
proxyProtocolJSON []byte,
followRedirects bool) error {
if reverseProxyId <= 0 {
return errors.New("invalid reverseProxyId")
}
op := NewReverseProxyOperator()
var op = NewReverseProxyOperator()
op.Id = reverseProxyId
if requestHostType < 0 {
@@ -329,6 +331,7 @@ func (this *ReverseProxyDAO) UpdateReverseProxy(tx *dbs.Tx,
op.RequestURI = requestURI
op.StripPrefix = stripPrefix
op.AutoFlush = autoFlush
op.FollowRedirects = followRedirects
if len(addHeaders) == 0 {
addHeaders = []string{}

View File

@@ -24,6 +24,7 @@ type ReverseProxy struct {
MaxConns uint32 `field:"maxConns"` // 最大并发连接数
MaxIdleConns uint32 `field:"maxIdleConns"` // 最大空闲连接数
ProxyProtocol string `field:"proxyProtocol"` // Proxy Protocol配置
FollowRedirects uint8 `field:"followRedirects"` // 回源跟随
}
type ReverseProxyOperator struct {
@@ -49,6 +50,7 @@ type ReverseProxyOperator struct {
MaxConns interface{} // 最大并发连接数
MaxIdleConns interface{} // 最大空闲连接数
ProxyProtocol interface{} // Proxy Protocol配置
FollowRedirects interface{} // 回源跟随
}
func NewReverseProxyOperator() *ReverseProxyOperator {

View File

@@ -1227,6 +1227,16 @@ func (this *ServerDAO) FindAllEnabledServerIdsWithSSLPolicyIds(tx *dbs.Tx, sslPo
return
}
// ExistEnabledUserServerWithSSLPolicyId 检查是否存在某个用户的策略
func (this *ServerDAO) ExistEnabledUserServerWithSSLPolicyId(tx *dbs.Tx, userId int64, sslPolicyId int64) (bool, error) {
return this.Query(tx).
State(ServerStateEnabled).
Attr("userId", userId).
Where("(JSON_CONTAINS(https, :jsonQuery) OR JSON_CONTAINS(tls, :jsonQuery))").
Param("jsonQuery", maps.Map{"sslPolicyRef": maps.Map{"sslPolicyId": sslPolicyId}}.AsJSON()).
Exist()
}
// CountEnabledServersWithWebIds 计算使用某个缓存策略的所有服务数量
func (this *ServerDAO) CountEnabledServersWithWebIds(tx *dbs.Tx, webIds []int64) (count int64, err error) {
if len(webIds) == 0 {

View File

@@ -1,6 +1,7 @@
package models
import (
"bytes"
"encoding/json"
"errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
@@ -113,6 +114,8 @@ func (this *SSLCertDAO) CreateCert(tx *dbs.Tx, adminId int64, userId int64, isOn
}
op.CommonNames = commonNamesJSON
op.OcspIsUpdated = false
err = this.Save(tx, op)
if err != nil {
return 0, err
@@ -121,11 +124,33 @@ func (this *SSLCertDAO) CreateCert(tx *dbs.Tx, adminId int64, userId int64, isOn
}
// UpdateCert 修改证书
func (this *SSLCertDAO) UpdateCert(tx *dbs.Tx, certId int64, isOn bool, name string, description string, serverName string, isCA bool, certData []byte, keyData []byte, timeBeginAt int64, timeEndAt int64, dnsNames []string, commonNames []string) error {
func (this *SSLCertDAO) UpdateCert(tx *dbs.Tx,
certId int64,
isOn bool,
name string,
description string,
serverName string,
isCA bool,
certData []byte,
keyData []byte,
timeBeginAt int64,
timeEndAt int64,
dnsNames []string, commonNames []string) error {
if certId <= 0 {
return errors.New("invalid certId")
}
op := NewSSLCertOperator()
oldOne, err := this.Query(tx).Find()
if err != nil {
return err
}
if oldOne == nil {
return nil
}
var oldCert = oldOne.(*SSLCert)
var dataIsChanged = bytes.Compare(certData, []byte(oldCert.CertData)) != 0 || bytes.Compare(keyData, []byte(oldCert.KeyData)) != 0
var op = NewSSLCertOperator()
op.Id = certId
op.IsOn = isOn
op.Name = name
@@ -156,6 +181,16 @@ func (this *SSLCertDAO) UpdateCert(tx *dbs.Tx, certId int64, isOn bool, name str
}
op.CommonNames = commonNamesJSON
// OCSP
if dataIsChanged {
op.OcspIsUpdated = 0
op.Ocsp = ""
op.OcspUpdatedAt = 0
op.OcspError = ""
op.OcspTries = 0
op.OcspExpiresAt = 0
}
err = this.Save(tx, op)
if err != nil {
return err
@@ -195,6 +230,13 @@ func (this *SSLCertDAO) ComposeCertConfig(tx *dbs.Tx, certId int64, cacheMap *ut
config.TimeBeginAt = int64(cert.TimeBeginAt)
config.TimeEndAt = int64(cert.TimeEndAt)
// OCSP
if int64(cert.OcspExpiresAt) > time.Now().Unix() {
config.OCSP = []byte(cert.Ocsp)
config.OCSPExpiresAt = int64(cert.OcspExpiresAt)
}
config.OCSPError = cert.OcspError
if IsNotNull(cert.DnsNames) {
dnsNames := []string{}
err := json.Unmarshal([]byte(cert.DnsNames), &dnsNames)
@@ -356,6 +398,197 @@ func (this *SSLCertDAO) CheckUserCert(tx *dbs.Tx, certId int64, userId int64) er
return nil
}
// ListCertsToUpdateOCSP 查找需要更新OCSP的证书
func (this *SSLCertDAO) ListCertsToUpdateOCSP(tx *dbs.Tx, maxTries int, size int64) (result []*SSLCert, err error) {
var nowTime = time.Now().Unix()
var query = this.Query(tx).
State(SSLCertStateEnabled).
Lt("ocspExpiresAt", nowTime+120). // 提前 N 秒钟准备更新
Lt("ocspTries", maxTries).
Lt("timeBeginAt", nowTime).
Gt("timeEndAt", nowTime)
// TODO 需要排除没有被server使用的policy或许可以增加一个字段记录policy最近使用时间
// 检查函数
var JSONArrayAggIsEnabled = false
_, err = this.Object().Instance.Exec("SELECT JSON_ARRAYAGG('1')")
if err == nil {
JSONArrayAggIsEnabled = true
}
if JSONArrayAggIsEnabled {
query.Where("JSON_CONTAINS((SELECT JSON_ARRAYAGG(JSON_EXTRACT(certs, '$[*].certId')) FROM edgeSSLPolicies WHERE state=1 AND ocspIsOn=1 AND certs IS NOT NULL), CAST(id AS CHAR))")
} else {
query.Where("JSON_CONTAINS((SELECT REPLACE(GROUP_CONCAT(JSON_EXTRACT(certs, '$[*].certId')), '],[', ',') FROM edgeSSLPolicies WHERE state=1 AND ocspIsOn=1 AND certs IS NOT NULL), CAST(id AS CHAR))")
}
_, err = query.
Asc("ocspUpdatedAt").
Limit(size).
Slice(&result).
FindAll()
return
}
// ListCertOCSPAfterVersion 列出某个版本后的OCSP
func (this *SSLCertDAO) ListCertOCSPAfterVersion(tx *dbs.Tx, version int64, size int64) (result []*SSLCert, err error) {
// 不需要判断ocsp是否为空
_, err = this.Query(tx).
Result("id", "ocsp", "ocspUpdatedVersion", "ocspExpiresAt").
State(SSLCertStateEnabled).
Attr("ocspIsUpdated", 1).
Gt("ocspUpdatedVersion", version).
Asc("ocspUpdatedVersion").
Limit(size).
Slice(&result).
FindAll()
return
}
// FindCertOCSPLatestVersion 获取OCSP最新版本
func (this *SSLCertDAO) FindCertOCSPLatestVersion(tx *dbs.Tx) (int64, error) {
return this.Query(tx).
Result("ocspUpdatedVersion").
Desc("ocspUpdatedVersion").
Limit(1).
FindInt64Col(0)
}
// PrepareCertOCSPUpdating 更新OCSP更新时间以便于准备更新相当于锁定
func (this *SSLCertDAO) PrepareCertOCSPUpdating(tx *dbs.Tx, certId int64) error {
return this.Query(tx).
Pk(certId).
Set("ocspUpdatedAt", time.Now().Unix()).
UpdateQuickly()
}
// UpdateCertOCSP 修改OCSP
func (this *SSLCertDAO) UpdateCertOCSP(tx *dbs.Tx, certId int64, ocsp []byte, expiresAt int64, hasErr bool, errString string) error {
if hasErr && len(errString) == 0 {
errString = "failed"
}
version, err := SharedSysLockerDAO.Increase(tx, "SSL_CERT_OCSP_VERSION", 1)
if err != nil {
return err
}
if ocsp == nil {
ocsp = []byte{}
}
// 限制长度
if len(errString) > 300 {
errString = errString[:300]
}
var query = this.Query(tx).
Pk(certId).
Set("ocsp", ocsp).
Set("ocspError", errString).
Set("ocspIsUpdated", true).
Set("ocspUpdatedAt", time.Now().Unix()).
Set("ocspUpdatedVersion", version).
Set("ocspExpiresAt", expiresAt)
if hasErr {
query.Set("ocspTries", dbs.SQL("ocspTries+1"))
} else {
query.Set("ocspTries", 0)
}
err = query.UpdateQuickly()
if err != nil {
return err
}
// 注意:这里不通知更新,避免频繁的更新导致服务不稳定
return nil
}
// CountAllSSLCertsWithOCSPError 计算有OCSP错误的证书数量
func (this *SSLCertDAO) CountAllSSLCertsWithOCSPError(tx *dbs.Tx, keyword string) (int64, error) {
var query = this.Query(tx)
if len(keyword) > 0 {
query.Where("(name LIKE :keyword OR description LIKE :keyword OR dnsNames LIKE :keyword OR commonNames LIKE :keyword OR ocspError LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
}
return query.
State(SSLCertStateEnabled).
Attr("ocspIsUpdated", true).
Where("LENGTH(ocspError) > 0").
Count()
}
// ListSSLCertsWithOCSPError 列出有OCSP错误的证书
func (this *SSLCertDAO) ListSSLCertsWithOCSPError(tx *dbs.Tx, keyword string, offset int64, size int64) (result []*SSLCert, err error) {
var query = this.Query(tx)
if len(keyword) > 0 {
query.Where("(name LIKE :keyword OR description LIKE :keyword OR dnsNames LIKE :keyword OR commonNames LIKE :keyword OR ocspError LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
}
_, err = query.
State(SSLCertStateEnabled).
Attr("ocspIsUpdated", true).
Where("LENGTH(ocspError) > 0").
Offset(offset).
Limit(size).
DescPk().
Slice(&result).
FindAll()
return
}
// IgnoreSSLCertsWithOCSPError 忽略一组OCSP证书错误
func (this *SSLCertDAO) IgnoreSSLCertsWithOCSPError(tx *dbs.Tx, certIds []int64) error {
for _, certId := range certIds {
err := this.Query(tx).
Pk(certId).
Set("ocspError", "").
UpdateQuickly()
if err != nil {
return err
}
}
return nil
}
// ResetSSLCertsWithOCSPError 重置一组证书OCSP错误状态
func (this *SSLCertDAO) ResetSSLCertsWithOCSPError(tx *dbs.Tx, certIds []int64) error {
for _, certId := range certIds {
err := this.Query(tx).
Pk(certId).
Set("ocspIsUpdated", 0).
Set("ocspUpdatedAt", 0).
Set("ocspError", "").
Set("ocspTries", 0).
UpdateQuickly()
if err != nil {
return err
}
}
return nil
}
// ResetAllSSLCertsWithOCSPError 重置所有证书OCSP错误状态
func (this *SSLCertDAO) ResetAllSSLCertsWithOCSPError(tx *dbs.Tx) error {
return this.Query(tx).
State(SSLCertStateEnabled).
Attr("ocspIsUpdated", 1).
Where("LENGTH(ocspError)>0").
Set("ocspIsUpdated", 0).
Set("ocspUpdatedAt", 0).
Set("ocspError", "").
Set("ocspTries", 0).
UpdateQuickly()
}
// NotifyUpdate 通知更新
func (this *SSLCertDAO) NotifyUpdate(tx *dbs.Tx, certId int64) error {
policyIds, err := SharedSSLPolicyDAO.FindAllEnabledPolicyIdsWithCertId(tx, certId)

View File

@@ -1,5 +1,19 @@
package models
package models_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
_ "github.com/go-sql-driver/mysql"
timeutil "github.com/iwind/TeaGo/utils/time"
"testing"
)
func TestSSLCertDAO_ListCertsToUpdateOCSP(t *testing.T) {
var dao = models.NewSSLCertDAO()
certs, err := dao.ListCertsToUpdateOCSP(nil, 3600, 10)
if err != nil {
t.Fatal(err)
}
for _, cert := range certs {
t.Log(cert.Id, cert.Name, "updatedAt:", cert.OcspUpdatedAt, timeutil.FormatTime("Y-m-d H:i:s", int64(cert.OcspUpdatedAt)), cert.OcspIsUpdated)
}
}

View File

@@ -1,52 +1,66 @@
package models
// SSL证书
// SSLCert SSL证书
type SSLCert struct {
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID
State uint8 `field:"state"` // 状态
CreatedAt uint64 `field:"createdAt"` // 创建时间
UpdatedAt uint64 `field:"updatedAt"` // 修改时间
IsOn uint8 `field:"isOn"` // 是否启用
Name string `field:"name"` // 证书名
Description string `field:"description"` // 描述
CertData string `field:"certData"` // 证书内容
KeyData string `field:"keyData"` // 密钥内容
ServerName string `field:"serverName"` // 证书使用的主机名
IsCA uint8 `field:"isCA"` // 是否为CA证书
GroupIds string `field:"groupIds"` // 证书分组
TimeBeginAt uint64 `field:"timeBeginAt"` // 开始时间
TimeEndAt uint64 `field:"timeEndAt"` // 结束时间
DnsNames string `field:"dnsNames"` // DNS名称列表
CommonNames string `field:"commonNames"` // 发行单位列表
IsACME uint8 `field:"isACME"` // 是否为ACME自动生成的
AcmeTaskId uint64 `field:"acmeTaskId"` // ACME任务ID
NotifiedAt uint64 `field:"notifiedAt"` // 最后通知时间
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID
State uint8 `field:"state"` // 状态
CreatedAt uint64 `field:"createdAt"` // 创建时间
UpdatedAt uint64 `field:"updatedAt"` // 修改时间
IsOn uint8 `field:"isOn"` // 是否启用
Name string `field:"name"` // 证书名
Description string `field:"description"` // 描述
CertData string `field:"certData"` // 证书内容
KeyData string `field:"keyData"` // 密钥内容
ServerName string `field:"serverName"` // 证书使用的主机名
IsCA uint8 `field:"isCA"` // 是否为CA证书
GroupIds string `field:"groupIds"` // 证书分组
TimeBeginAt uint64 `field:"timeBeginAt"` // 开始时间
TimeEndAt uint64 `field:"timeEndAt"` // 结束时间
DnsNames string `field:"dnsNames"` // DNS名称列表
CommonNames string `field:"commonNames"` // 发行单位列表
IsACME uint8 `field:"isACME"` // 是否为ACME自动生成的
AcmeTaskId uint64 `field:"acmeTaskId"` // ACME任务ID
NotifiedAt uint64 `field:"notifiedAt"` // 最后通知时间
Ocsp string `field:"ocsp"` // OCSP缓存
OcspIsUpdated uint8 `field:"ocspIsUpdated"` // OCSP是否已更新
OcspUpdatedAt uint64 `field:"ocspUpdatedAt"` // OCSP更新时间
OcspError string `field:"ocspError"` // OCSP更新错误
OcspUpdatedVersion uint64 `field:"ocspUpdatedVersion"` // OCSP更新版本
OcspExpiresAt uint64 `field:"ocspExpiresAt"` // OCSP过期时间(UTC)
OcspTries uint32 `field:"ocspTries"` // OCSP尝试次数
}
type SSLCertOperator struct {
Id interface{} // ID
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
State interface{} // 状态
CreatedAt interface{} // 创建时间
UpdatedAt interface{} // 修改时间
IsOn interface{} // 是否启用
Name interface{} // 证书名
Description interface{} // 描述
CertData interface{} // 证书内容
KeyData interface{} // 密钥内容
ServerName interface{} // 证书使用的主机名
IsCA interface{} // 是否为CA证书
GroupIds interface{} // 证书分组
TimeBeginAt interface{} // 开始时间
TimeEndAt interface{} // 结束时间
DnsNames interface{} // DNS名称列表
CommonNames interface{} // 发行单位列表
IsACME interface{} // 是否为ACME自动生成的
AcmeTaskId interface{} // ACME任务ID
NotifiedAt interface{} // 最后通知时间
Id interface{} // ID
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
State interface{} // 状态
CreatedAt interface{} // 创建时间
UpdatedAt interface{} // 修改时间
IsOn interface{} // 是否启用
Name interface{} // 证书名
Description interface{} // 描述
CertData interface{} // 证书内容
KeyData interface{} // 密钥内容
ServerName interface{} // 证书使用的主机名
IsCA interface{} // 是否为CA证书
GroupIds interface{} // 证书分组
TimeBeginAt interface{} // 开始时间
TimeEndAt interface{} // 结束时间
DnsNames interface{} // DNS名称列表
CommonNames interface{} // 发行单位列表
IsACME interface{} // 是否为ACME自动生成的
AcmeTaskId interface{} // ACME任务ID
NotifiedAt interface{} // 最后通知时间
Ocsp interface{} // OCSP缓存
OcspIsUpdated interface{} // OCSP是否已更新
OcspUpdatedAt interface{} // OCSP更新时间
OcspError interface{} // OCSP更新错误
OcspUpdatedVersion interface{} // OCSP更新版本
OcspExpiresAt interface{} // OCSP过期时间(UTC)
OcspTries interface{} // OCSP尝试次数
}
func NewSSLCertOperator() *SSLCertOperator {

View File

@@ -1 +1,31 @@
package models
import "encoding/json"
func (this *SSLCert) DecodeDNSNames() []string {
if len(this.DnsNames) == 0 {
return nil
}
var result = []string{}
var err = json.Unmarshal([]byte(this.DnsNames), &result)
if err != nil {
return nil
}
return result
}
func (this *SSLCert) DecodeCommonNames() []string {
if len(this.CommonNames) == 0 {
return nil
}
var result = []string{}
var err = json.Unmarshal([]byte(this.CommonNames), &result)
if err != nil {
return nil
}
return result
}

View File

@@ -167,6 +167,9 @@ func (this *SSLPolicyDAO) ComposePolicyConfig(tx *dbs.Tx, policyId int64, cacheM
config.HSTS = hstsConfig
}
// ocsp
config.OCSPIsOn = policy.OcspIsOn == 1
if cacheMap != nil {
cacheMap.Put(cacheKey, config)
}
@@ -196,7 +199,7 @@ func (this *SSLPolicyDAO) FindAllEnabledPolicyIdsWithCertId(tx *dbs.Tx, certId i
}
// CreatePolicy 创建Policy
func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) (int64, error) {
func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, ocspIsOn bool, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) (int64, error) {
op := NewSSLPolicyOperator()
op.State = SSLPolicyStateEnabled
op.IsOn = true
@@ -213,6 +216,8 @@ func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64,
op.Hsts = hstsJSON
}
op.OcspIsOn = ocspIsOn
op.ClientAuthType = clientAuthType
if len(clientCACertsJSON) > 0 {
op.ClientCACerts = clientCACertsJSON
@@ -234,7 +239,7 @@ func (this *SSLPolicyDAO) CreatePolicy(tx *dbs.Tx, adminId int64, userId int64,
}
// UpdatePolicy 修改Policy
func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) error {
func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled bool, minVersion string, certsJSON []byte, hstsJSON []byte, ocspIsOn bool, clientAuthType int32, clientCACertsJSON []byte, cipherSuitesIsOn bool, cipherSuites []string) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -251,6 +256,8 @@ func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled
op.Hsts = hstsJSON
}
op.OcspIsOn = ocspIsOn
op.ClientAuthType = clientAuthType
if len(clientCACertsJSON) > 0 {
op.ClientCACerts = clientCACertsJSON
@@ -274,9 +281,9 @@ func (this *SSLPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, http2Enabled
}
// CheckUserPolicy 检查是否为用户所属策略
func (this *SSLPolicyDAO) CheckUserPolicy(tx *dbs.Tx, policyId int64, userId int64) error {
func (this *SSLPolicyDAO) CheckUserPolicy(tx *dbs.Tx, userId int64, policyId int64) error {
if policyId <= 0 || userId <= 0 {
return errors.New("not found")
return ErrNotFound
}
ok, err := this.Query(tx).
State(SSLPolicyStateEnabled).
@@ -287,7 +294,14 @@ func (this *SSLPolicyDAO) CheckUserPolicy(tx *dbs.Tx, policyId int64, userId int
return err
}
if !ok {
return errors.New("not found")
// 是否为当前用户的某个服务所用
exists, err := SharedServerDAO.ExistEnabledUserServerWithSSLPolicyId(tx, userId, policyId)
if err != nil {
return err
}
if !exists {
return ErrNotFound
}
}
return nil
}

View File

@@ -1,6 +1,6 @@
package models
//
// SSLPolicy SSL配置策略
type SSLPolicy struct {
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
@@ -14,6 +14,7 @@ type SSLPolicy struct {
CipherSuites string `field:"cipherSuites"` // 加密算法套件
Hsts string `field:"hsts"` // HSTS设置
Http2Enabled uint8 `field:"http2Enabled"` // 是否启用HTTP/2
OcspIsOn uint8 `field:"ocspIsOn"` // 是否启用OCSP
State uint8 `field:"state"` // 状态
CreatedAt uint64 `field:"createdAt"` // 创建时间
}
@@ -31,6 +32,7 @@ type SSLPolicyOperator struct {
CipherSuites interface{} // 加密算法套件
Hsts interface{} // HSTS设置
Http2Enabled interface{} // 是否启用HTTP/2
OcspIsOn interface{} // 是否启用OCSP
State interface{} // 状态
CreatedAt interface{} // 创建时间
}

View File

@@ -109,10 +109,13 @@ func (this *ServerHTTPFirewallDailyStatDAO) GroupDailyCount(tx *dbs.Tx, userId i
}
// FindDailyStats 查询某个日期段内的记录
func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, action string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) {
func (this *ServerHTTPFirewallDailyStatDAO) FindDailyStats(tx *dbs.Tx, userId int64, serverId int64, actions []string, dayFrom string, dayTo string) (result []*ServerHTTPFirewallDailyStat, err error) {
if len(actions) == 0 {
return nil, nil
}
query := this.Query(tx).
Between("day", dayFrom, dayTo).
Attr("action", action)
Attr("action", actions)
if serverId > 0 {
query.Attr("serverId", serverId)
} else if userId > 0 {

View File

@@ -150,13 +150,13 @@ func (this *AliDNSProvider) AddRecord(domain string, newRecord *dnstypes.Record)
resp := alidns.CreateAddDomainRecordResponse()
err := this.doAPI(req, resp)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
if resp.IsSuccess() {
return nil
}
return errors.New(resp.GetHttpContentString())
return this.WrapError(errors.New(resp.GetHttpContentString()), domain, newRecord)
}
// UpdateRecord 修改记录
@@ -174,7 +174,7 @@ func (this *AliDNSProvider) UpdateRecord(domain string, record *dnstypes.Record,
resp := alidns.CreateUpdateDomainRecordResponse()
err := this.doAPI(req, resp)
return err
return this.WrapError(err, domain, newRecord)
}
// DeleteRecord 删除记录
@@ -184,7 +184,7 @@ func (this *AliDNSProvider) DeleteRecord(domain string, record *dnstypes.Record)
resp := alidns.CreateDeleteDomainRecordResponse()
err := this.doAPI(req, resp)
return err
return this.WrapError(err, domain, record)
}
// DefaultRoute 默认线路

View File

@@ -1,4 +1,29 @@
package dnsclients
import (
"errors"
"github.com/TeaOSLab/EdgeAPI/internal/dnsclients/dnstypes"
"github.com/iwind/TeaGo/types"
)
type BaseProvider struct {
}
// WrapError 封装解析相关错误
func (this *BaseProvider) WrapError(err error, domain string, record *dnstypes.Record) error {
if err == nil {
return nil
}
if record == nil {
return err
}
var fullname = ""
if len(record.Name) == 0 {
fullname = domain
} else {
fullname = record.Name + "." + domain
}
return errors.New("record operation failed: '" + fullname + " " + record.Type + " " + record.Value + " " + types.String(record.TTL) + "': " + err.Error())
}

View File

@@ -0,0 +1,38 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package dnsclients_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/dnsclients"
"github.com/TeaOSLab/EdgeAPI/internal/dnsclients/dnstypes"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"testing"
)
func TestBaseProvider_WrapError(t *testing.T) {
var provider = &dnsclients.BaseProvider{}
t.Log(provider.WrapError(nil, "example.com", &dnstypes.Record{
Id: "",
Name: "a",
Type: "A",
Value: "192.168.1.100",
Route: "",
TTL: 3600,
}))
t.Log(provider.WrapError(errors.New("fake error"), "example.com", &dnstypes.Record{
Id: "",
Name: "a",
Type: "A",
Value: "192.168.1.100",
Route: "",
TTL: 3600,
}))
t.Log(provider.WrapError(errors.New("fake error"), "example.com", &dnstypes.Record{
Id: "",
Name: "",
Type: "A",
Value: "192.168.1.100",
Route: "",
TTL: 3600,
}))
}

View File

@@ -169,7 +169,7 @@ func (this *CloudFlareProvider) QueryRecord(domain string, name string, recordTy
func (this *CloudFlareProvider) AddRecord(domain string, newRecord *dnstypes.Record) error {
zoneId, err := this.findZoneIdWithDomain(domain)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
resp := new(cloudflare.CreateDNSRecordResponse)
@@ -186,7 +186,7 @@ func (this *CloudFlareProvider) AddRecord(domain string, newRecord *dnstypes.Rec
"ttl": ttl,
}, resp)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
return nil
}
@@ -195,7 +195,7 @@ func (this *CloudFlareProvider) AddRecord(domain string, newRecord *dnstypes.Rec
func (this *CloudFlareProvider) UpdateRecord(domain string, record *dnstypes.Record, newRecord *dnstypes.Record) error {
zoneId, err := this.findZoneIdWithDomain(domain)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
var ttl = newRecord.TTL
@@ -216,13 +216,13 @@ func (this *CloudFlareProvider) UpdateRecord(domain string, record *dnstypes.Rec
func (this *CloudFlareProvider) DeleteRecord(domain string, record *dnstypes.Record) error {
zoneId, err := this.findZoneIdWithDomain(domain)
if err != nil {
return err
return this.WrapError(err, domain, record)
}
resp := new(cloudflare.DeleteDNSRecordResponse)
err = this.doAPI(http.MethodDelete, "zones/"+zoneId+"/dns_records/"+record.Id, map[string]string{}, nil, resp)
if err != nil {
return err
return this.WrapError(err, domain, record)
}
return nil
}

View File

@@ -29,6 +29,8 @@ var customHTTPClient = &http.Client{
type CustomHTTPProvider struct {
url string
secret string
BaseProvider
}
// Auth 认证
@@ -96,7 +98,7 @@ func (this *CustomHTTPProvider) QueryRecord(domain string, name string, recordTy
if err != nil {
return nil, err
}
if len(resp) == 0 {
if len(resp) == 0 || string(resp) == "null" {
return nil, nil
}
record := &dnstypes.Record{}
@@ -117,7 +119,7 @@ func (this *CustomHTTPProvider) AddRecord(domain string, newRecord *dnstypes.Rec
"domain": domain,
"newRecord": newRecord,
})
return err
return this.WrapError(err, domain, newRecord)
}
// UpdateRecord 修改记录
@@ -128,7 +130,7 @@ func (this *CustomHTTPProvider) UpdateRecord(domain string, record *dnstypes.Rec
"record": record,
"newRecord": newRecord,
})
return err
return this.WrapError(err, domain, newRecord)
}
// DeleteRecord 删除记录
@@ -138,7 +140,7 @@ func (this *CustomHTTPProvider) DeleteRecord(domain string, record *dnstypes.Rec
"domain": domain,
"record": record,
})
return err
return this.WrapError(err, domain, record)
}
// DefaultRoute 默认线路
@@ -162,10 +164,11 @@ func (this *CustomHTTPProvider) post(params maps.Map) (respData []byte, err erro
if err != nil {
return nil, err
}
timestamp := strconv.FormatInt(time.Now().Unix(), 10)
var timestamp = strconv.FormatInt(time.Now().Unix(), 10)
req.Header.Set("Timestamp", timestamp)
req.Header.Set("Token", fmt.Sprintf("%x", sha1.Sum([]byte(this.secret+"@"+timestamp))))
req.Header.Set("User-Agent", teaconst.ProductName+"/"+teaconst.Version)
req.Header.Set("Content-Type", "application/json")
resp, err := customHTTPClient.Do(req)
if err != nil {

View File

@@ -182,7 +182,7 @@ func (this *DNSPodProvider) AddRecord(domain string, newRecord *dnstypes.Record)
args["ttl"] = types.String(newRecord.TTL)
}
_, err := this.post("/Record.Create", args)
return err
return this.WrapError(err, domain, newRecord)
}
// UpdateRecord 修改记录
@@ -211,7 +211,7 @@ func (this *DNSPodProvider) UpdateRecord(domain string, record *dnstypes.Record,
args["ttl"] = types.String(newRecord.TTL)
}
_, err := this.post("/Record.Modify", args)
return err
return this.WrapError(err, domain, newRecord)
}
// DeleteRecord 删除记录
@@ -225,7 +225,7 @@ func (this *DNSPodProvider) DeleteRecord(domain string, record *dnstypes.Record)
"record_id": record.Id,
})
return err
return this.WrapError(err, domain, record)
}
// 发送请求

View File

@@ -1331,7 +1331,7 @@ func (this *HuaweiDNSProvider) QueryRecord(domain string, name string, recordTyp
func (this *HuaweiDNSProvider) AddRecord(domain string, newRecord *dnstypes.Record) error {
zoneId, err := this.findZoneIdWithDomain(domain)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
var resp = new(huaweidns.ZonesCreateRecordSetResponse)
@@ -1354,7 +1354,7 @@ func (this *HuaweiDNSProvider) AddRecord(domain string, newRecord *dnstypes.Reco
"ttl": ttl,
}, resp)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
newRecord.Id = resp.Id + "@" + newRecord.Value
@@ -1366,7 +1366,7 @@ func (this *HuaweiDNSProvider) AddRecord(domain string, newRecord *dnstypes.Reco
func (this *HuaweiDNSProvider) UpdateRecord(domain string, record *dnstypes.Record, newRecord *dnstypes.Record) error {
zoneId, err := this.findZoneIdWithDomain(domain)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
var recordId string
@@ -1397,7 +1397,7 @@ func (this *HuaweiDNSProvider) UpdateRecord(domain string, record *dnstypes.Reco
"ttl": ttl,
}, resp)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
return nil
@@ -1407,7 +1407,7 @@ func (this *HuaweiDNSProvider) UpdateRecord(domain string, record *dnstypes.Reco
func (this *HuaweiDNSProvider) DeleteRecord(domain string, record *dnstypes.Record) error {
zoneId, err := this.findZoneIdWithDomain(domain)
if err != nil {
return err
return this.WrapError(err, domain, record)
}
var recordId string
@@ -1421,7 +1421,7 @@ func (this *HuaweiDNSProvider) DeleteRecord(domain string, record *dnstypes.Reco
var resp = new(huaweidns.ZonesDeleteRecordSetResponse)
err = this.doAPI(http.MethodDelete, "/v2.1/zones/"+zoneId+"/recordsets/"+recordId, map[string]string{}, maps.Map{}, resp)
if err != nil {
return err
return this.WrapError(err, domain, record)
}
return nil

View File

@@ -17,6 +17,8 @@ import (
type LocalEdgeDNSProvider struct {
clusterId int64 // 集群ID
ttl int32 // TTL
BaseProvider
}
// Auth 认证
@@ -193,10 +195,10 @@ func (this *LocalEdgeDNSProvider) AddRecord(domain string, newRecord *dnstypes.R
var tx *dbs.Tx
domainId, err := nameservers.SharedNSDomainDAO.FindDomainIdWithName(tx, this.clusterId, domain)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
if domainId == 0 {
return errors.New("can not find domain '" + domain + "'")
return this.WrapError(errors.New("can not find domain '"+domain+"'"), domain, newRecord)
}
var routeIds = []string{}
@@ -209,7 +211,7 @@ func (this *LocalEdgeDNSProvider) AddRecord(domain string, newRecord *dnstypes.R
}
_, err = nameservers.SharedNSRecordDAO.CreateRecord(tx, domainId, "", newRecord.Name, newRecord.Type, newRecord.Value, newRecord.TTL, routeIds)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
return nil
@@ -220,7 +222,7 @@ func (this *LocalEdgeDNSProvider) UpdateRecord(domain string, record *dnstypes.R
var tx *dbs.Tx
domainId, err := nameservers.SharedNSDomainDAO.FindDomainIdWithName(tx, this.clusterId, domain)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
if domainId == 0 {
return errors.New("can not find domain '" + domain + "'")
@@ -238,17 +240,17 @@ func (this *LocalEdgeDNSProvider) UpdateRecord(domain string, record *dnstypes.R
if len(record.Id) > 0 {
err = nameservers.SharedNSRecordDAO.UpdateRecord(tx, types.Int64(record.Id), "", newRecord.Name, newRecord.Type, newRecord.Value, newRecord.TTL, routeIds, true)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
} else {
realRecord, err := nameservers.SharedNSRecordDAO.FindEnabledRecordWithName(tx, domainId, record.Name, record.Type)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
if realRecord != nil {
err = nameservers.SharedNSRecordDAO.UpdateRecord(tx, types.Int64(realRecord.Id), "", newRecord.Name, newRecord.Type, newRecord.Value, newRecord.TTL, routeIds, true)
if err != nil {
return err
return this.WrapError(err, domain, newRecord)
}
}
}
@@ -261,7 +263,7 @@ func (this *LocalEdgeDNSProvider) DeleteRecord(domain string, record *dnstypes.R
var tx *dbs.Tx
domainId, err := nameservers.SharedNSDomainDAO.FindDomainIdWithName(tx, this.clusterId, domain)
if err != nil {
return err
return this.WrapError(err, domain, record)
}
if domainId == 0 {
return errors.New("can not find domain '" + domain + "'")
@@ -270,17 +272,17 @@ func (this *LocalEdgeDNSProvider) DeleteRecord(domain string, record *dnstypes.R
if len(record.Id) > 0 {
err = nameservers.SharedNSRecordDAO.DisableNSRecord(tx, types.Int64(record.Id))
if err != nil {
return err
return this.WrapError(err, domain, record)
}
} else {
realRecord, err := nameservers.SharedNSRecordDAO.FindEnabledRecordWithName(tx, domainId, record.Name, record.Type)
if err != nil {
return err
return this.WrapError(err, domain, record)
}
if realRecord != nil {
err = nameservers.SharedNSRecordDAO.DisableNSRecord(tx, types.Int64(realRecord.Id))
if err != nil {
return err
return this.WrapError(err, domain, record)
}
}
}

View File

@@ -15,7 +15,6 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/rpc"
"github.com/TeaOSLab/EdgeAPI/internal/setup"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/go-yaml/yaml"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
@@ -26,6 +25,7 @@ import (
"github.com/iwind/gosock/pkg/gosock"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"gopkg.in/yaml.v3"
"io/ioutil"
"log"
"net"
@@ -37,6 +37,9 @@ import (
"strconv"
"sync"
"time"
// grpc decompression
_ "google.golang.org/grpc/encoding/gzip"
)
var sharedAPIConfig *configs.APIConfig = nil
@@ -276,7 +279,7 @@ func (this *APINode) autoUpgrade() error {
if err != nil {
return errors.New("decode database config failed: " + err.Error())
}
dbConfig := config.DBs[Tea.Env]
var dbConfig = config.DBs[Tea.Env]
db, err := dbs.NewInstanceFromConfig(dbConfig)
if err != nil {
return errors.New("load database failed: " + err.Error())
@@ -287,8 +290,8 @@ func (this *APINode) autoUpgrade() error {
}
if one != nil {
// 如果是同样的版本,则直接认为是最新版本
version := one.GetString("version")
if stringutil.VersionCompare(version, teaconst.Version) >= 0 {
var version = one.GetString("version")
if stringutil.VersionCompare(version, setup.ComposeSQLVersion()) >= 0 {
return nil
}
}

View File

@@ -9,8 +9,8 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/iwind/TeaGo/lists"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/disk"
"github.com/shirou/gopsutil/v3/cpu"
"github.com/shirou/gopsutil/v3/disk"
"os"
"runtime"
"strings"

View File

@@ -1,7 +1,7 @@
package nodes
import (
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/v3/cpu"
"testing"
"time"
)

View File

@@ -4,8 +4,8 @@ package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/shirou/gopsutil/load"
"github.com/shirou/gopsutil/mem"
"github.com/shirou/gopsutil/v3/load"
"github.com/shirou/gopsutil/v3/mem"
)
// 更新内存

View File

@@ -4,8 +4,8 @@ package nodes
import (
"context"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/mem"
"github.com/shirou/gopsutil/v3/cpu"
"github.com/shirou/gopsutil/v3/mem"
"math"
"sync"
"time"

View File

@@ -9,14 +9,12 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/tasks"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -743,7 +741,11 @@ func (this *AdminService) ComposeAdminDashboard(ctx context.Context, req *pb.Com
// 域名排行
this.BeginTag(ctx, "SharedServerDomainHourlyStatDAO.FindTopDomainStats")
topDomainStats, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStats(tx, hourFrom, hourTo, 10)
var topDomainStats []*stats.ServerDomainHourlyStat
topDomainStatsCache, ok := tasks.SharedCacheTaskManager.GetGlobalTopDomains()
if ok {
topDomainStats = topDomainStatsCache.([]*stats.ServerDomainHourlyStat)
}
this.EndTag(ctx, "SharedServerDomainHourlyStatDAO.FindTopDomainStats")
if err != nil {
return nil, err
@@ -816,7 +818,11 @@ func (this *AdminService) ComposeAdminDashboard(ctx context.Context, req *pb.Com
// 指标数据
this.BeginTag(ctx, "findMetricDataCharts")
pbCharts, err := this.findMetricDataCharts(tx)
var pbCharts []*pb.MetricDataChart
pbChartsCache, ok := tasks.SharedCacheTaskManager.Get(tasks.CacheKeyFindAllMetricDataCharts)
if ok {
pbCharts = pbChartsCache.([]*pb.MetricDataChart)
}
this.EndTag(ctx, "findMetricDataCharts")
if err != nil {
return nil, err
@@ -839,111 +845,3 @@ func (this *AdminService) UpdateAdminTheme(ctx context.Context, req *pb.UpdateAd
}
return this.Success()
}
// 查找集群、节点和服务的指标数据
func (this *AdminService) findMetricDataCharts(tx *dbs.Tx) (result []*pb.MetricDataChart, err error) {
// 集群指标
items, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, serverconfigs.MetricItemCategoryHTTP, nil)
if err != nil {
return nil, err
}
var pbMetricCharts = []*pb.MetricDataChart{}
for _, item := range items {
var itemId = int64(item.Id)
charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
if err != nil {
return nil, err
}
for _, chart := range charts {
if chart.IsOn == 0 {
continue
}
var pbChart = &pb.MetricChart{
Id: int64(chart.Id),
Name: chart.Name,
Type: chart.Type,
WidthDiv: chart.WidthDiv,
ParamsJSON: nil,
IsOn: chart.IsOn == 1,
MaxItems: types.Int32(chart.MaxItems),
MetricItem: &pb.MetricItem{
Id: itemId,
PeriodUnit: item.PeriodUnit,
Period: types.Int32(item.Period),
Name: item.Name,
Value: item.Value,
Category: item.Category,
Keys: item.DecodeKeys(),
Code: item.Code,
IsOn: item.IsOn == 1,
},
}
var pbStats = []*pb.MetricStat{}
switch chart.Type {
case serverconfigs.MetricChartTypeTimeLine:
itemStats, err := models.SharedMetricStatDAO.FindLatestItemStats(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
if err != nil {
return nil, err
}
for _, stat := range itemStats {
// 当前时间总和
count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
default:
itemStats, err := models.SharedMetricStatDAO.FindItemStatsAtLastTime(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
if err != nil {
return nil, err
}
for _, stat := range itemStats {
count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
}
pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
MetricChart: pbChart,
MetricStats: pbStats,
})
}
}
return pbMetricCharts, nil
}

View File

@@ -9,12 +9,12 @@ import (
"strings"
)
// 数据库相关服务
// DBService 数据库相关服务
type DBService struct {
BaseService
}
// 获取所有表信息
// FindAllDBTables 获取所有表信息
func (this *DBService) FindAllDBTables(ctx context.Context, req *pb.FindAllDBTablesRequest) (*pb.FindAllDBTablesResponse, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
@@ -60,7 +60,7 @@ func (this *DBService) FindAllDBTables(ctx context.Context, req *pb.FindAllDBTab
return &pb.FindAllDBTablesResponse{DbTables: pbTables}, nil
}
// 删除表
// DeleteDBTable 删除表
func (this *DBService) DeleteDBTable(ctx context.Context, req *pb.DeleteDBTableRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
@@ -84,7 +84,7 @@ func (this *DBService) DeleteDBTable(ctx context.Context, req *pb.DeleteDBTableR
return this.Success()
}
// 清空表
// TruncateDBTable 清空表
func (this *DBService) TruncateDBTable(ctx context.Context, req *pb.TruncateDBTableRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {

View File

@@ -12,6 +12,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"sort"
"strconv"
"time"
)
@@ -118,7 +119,7 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
return nil, err
}
{
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "log", dayFrom, day)
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, []string{"log", "tag"}, dayFrom, day)
if err != nil {
return nil, err
}
@@ -131,7 +132,7 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
}
}
{
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "captcha", dayFrom, day)
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, []string{"captcha"}, dayFrom, day)
if err != nil {
return nil, err
}
@@ -144,7 +145,7 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
}
}
{
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, "block", dayFrom, day)
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, 0, 0, []string{"block", "page"}, dayFrom, day)
if err != nil {
return nil, err
}
@@ -158,10 +159,14 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
}
// 规则分组
groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, 0, 0, dayFrom, day, 0, 10)
var today = timeutil.Format("Ymd")
groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, 0, 0, today, today, 0, 20)
if err != nil {
return nil, err
}
// 合并同名
var groupNamedStatsMap = map[string]*stats.ServerHTTPFirewallDailyStat{} // name => *ServerHTTPFirewallDailyStat
for _, stat := range groupStats {
ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId))
if err != nil {
@@ -171,11 +176,26 @@ func (this *FirewallService) ComposeFirewallGlobalBoard(ctx context.Context, req
continue
}
namedStat, ok := groupNamedStatsMap[ruleGroupName]
if ok {
namedStat.Count += stat.Count
} else {
groupNamedStatsMap[ruleGroupName] = stat
}
}
for ruleGroupName, stat := range groupNamedStatsMap {
result.HttpFirewallRuleGroups = append(result.HttpFirewallRuleGroups, &pb.ComposeFirewallGlobalBoardResponse_HTTPFirewallRuleGroupStat{
HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName},
Count: int64(stat.Count),
})
}
sort.Slice(result.HttpFirewallRuleGroups, func(i, j int) bool {
return result.HttpFirewallRuleGroups[i].Count > result.HttpFirewallRuleGroups[j].Count
})
if len(result.HttpFirewallRuleGroups) > 10 {
result.HttpFirewallRuleGroups = result.HttpFirewallRuleGroups[:10]
}
// 节点排行
topNodeStats, err := stats.SharedNodeTrafficHourlyStatDAO.FindTopNodeStatsWithAttack(tx, "node", hourFrom, hourTo, 10)

View File

@@ -72,7 +72,7 @@ func (this *HTTPAccessLogService) ListHTTPAccessLogs(ctx context.Context, req *p
}
}
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
accessLogs, requestId, hasMore, err := models.SharedHTTPAccessLogDAO.ListAccessLogs(tx, req.RequestId, req.Size, req.Day, req.HourFrom, req.HourTo, req.NodeClusterId, req.NodeId, req.ServerId, req.Reverse, req.HasError, req.FirewallPolicyId, req.FirewallRuleGroupId, req.FirewallRuleSetId, req.HasFirewallPolicy, req.UserId, req.Keyword, req.Ip, req.Domain)
if err != nil {
return nil, err
}

View File

@@ -46,7 +46,7 @@ func (this *HTTPCachePolicyService) CreateHTTPCachePolicy(ctx context.Context, r
tx := this.NullTx()
policyId, err := models.SharedHTTPCachePolicyDAO.CreateCachePolicy(tx, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON)
policyId, err := models.SharedHTTPCachePolicyDAO.CreateCachePolicy(tx, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache)
if err != nil {
return nil, err
}
@@ -63,7 +63,7 @@ func (this *HTTPCachePolicyService) UpdateHTTPCachePolicy(ctx context.Context, r
tx := this.NullTx()
err = models.SharedHTTPCachePolicyDAO.UpdateCachePolicy(tx, req.HttpCachePolicyId, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON)
err = models.SharedHTTPCachePolicyDAO.UpdateCachePolicy(tx, req.HttpCachePolicyId, req.IsOn, req.Name, req.Description, req.CapacityJSON, req.MaxKeys, req.MaxSizeJSON, req.Type, req.OptionsJSON, req.SyncCompressionCache)
if err != nil {
return nil, err
}
@@ -99,7 +99,7 @@ func (this *HTTPCachePolicyService) CountAllEnabledHTTPCachePolicies(ctx context
tx := this.NullTx()
count, err := models.SharedHTTPCachePolicyDAO.CountAllEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword)
count, err := models.SharedHTTPCachePolicyDAO.CountAllEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword, req.Type)
if err != nil {
return nil, err
}
@@ -116,7 +116,7 @@ func (this *HTTPCachePolicyService) ListEnabledHTTPCachePolicies(ctx context.Con
tx := this.NullTx()
cachePolicies, err := models.SharedHTTPCachePolicyDAO.ListEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword, req.Offset, req.Size)
cachePolicies, err := models.SharedHTTPCachePolicyDAO.ListEnabledHTTPCachePolicies(tx, req.NodeClusterId, req.Keyword, req.Type, req.Offset, req.Size)
if err != nil {
return nil, err
}

View File

@@ -578,6 +578,7 @@ func (this *NodeService) FindEnabledNode(ctx context.Context, req *pb.FindEnable
NodeRegion: pbRegion,
MaxCacheDiskCapacity: pbMaxCacheDiskCapacity,
MaxCacheMemoryCapacity: pbMaxCacheMemoryCapacity,
CacheDiskDir: node.CacheDiskDir,
}}, nil
}
@@ -1570,7 +1571,7 @@ func (this *NodeService) UpdateNodeCache(ctx context.Context, req *pb.UpdateNode
}
}
err = models.SharedNodeDAO.UpdateNodeCache(tx, req.NodeId, maxCacheDiskCapacityJSON, maxCacheMemoryCapacityJSON)
err = models.SharedNodeDAO.UpdateNodeCache(tx, req.NodeId, maxCacheDiskCapacityJSON, maxCacheMemoryCapacityJSON, req.CacheDiskDir)
if err != nil {
return nil, err
}

View File

@@ -298,6 +298,7 @@ func (this *NodeClusterService) ListEnabledNodeClusters(ctx context.Context, req
DnsDomainId: int64(cluster.DnsDomainId),
IsOn: cluster.IsOn == 1,
TimeZone: cluster.TimeZone,
IsPinned: cluster.IsPinned == 1,
})
}
@@ -1026,3 +1027,19 @@ func (this *NodeClusterService) FindEnabledNodeClusterConfigInfo(ctx context.Con
return result, nil
}
// UpdateNodeClusterPinned 设置集群是否置顶
func (this *NodeClusterService) UpdateNodeClusterPinned(ctx context.Context, req *pb.UpdateNodeClusterPinnedRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedNodeClusterDAO.UpdateClusterIsPinned(tx, req.NodeClusterId, req.IsPinned)
if err != nil {
return nil, err
}
return this.Success()
}

View File

@@ -72,7 +72,7 @@ func (this *OriginService) CreateOrigin(ctx context.Context, req *pb.CreateOrigi
}
}
originId, err := models.SharedOriginDAO.CreateOrigin(tx, adminId, userId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains)
originId, err := models.SharedOriginDAO.CreateOrigin(tx, adminId, userId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains, req.Host)
if err != nil {
return nil, err
}
@@ -139,7 +139,7 @@ func (this *OriginService) UpdateOrigin(ctx context.Context, req *pb.UpdateOrigi
}
}
err = models.SharedOriginDAO.UpdateOrigin(tx, req.OriginId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains)
err = models.SharedOriginDAO.UpdateOrigin(tx, req.OriginId, req.Name, string(addrMap.AsJSON()), req.Description, req.Weight, req.IsOn, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, certRef, req.Domains, req.Host)
if err != nil {
return nil, err
}

View File

@@ -216,7 +216,7 @@ func (this *ReverseProxyService) UpdateReverseProxy(ctx context.Context, req *pb
}
}
err = models.SharedReverseProxyDAO.UpdateReverseProxy(tx, req.ReverseProxyId, types.Int8(req.RequestHostType), req.RequestHost, req.RequestURI, req.StripPrefix, req.AutoFlush, req.AddHeaders, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, req.ProxyProtocolJSON)
err = models.SharedReverseProxyDAO.UpdateReverseProxy(tx, req.ReverseProxyId, types.Int8(req.RequestHostType), req.RequestHost, req.RequestURI, req.StripPrefix, req.AutoFlush, req.AddHeaders, connTimeout, readTimeout, idleTimeout, req.MaxConns, req.MaxIdleConns, req.ProxyProtocolJSON, req.FollowRedirects)
if err != nil {
return nil, err
}

View File

@@ -42,7 +42,7 @@ func (this *ServerService) CreateServer(ctx context.Context, req *pb.CreateServe
return nil, err
}
if httpsConfig.SSLPolicyRef != nil && httpsConfig.SSLPolicyRef.SSLPolicyId > 0 {
err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, httpsConfig.SSLPolicyRef.SSLPolicyId, userId)
err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, userId, httpsConfig.SSLPolicyRef.SSLPolicyId)
if err != nil {
return nil, err
}
@@ -57,7 +57,7 @@ func (this *ServerService) CreateServer(ctx context.Context, req *pb.CreateServe
return nil, err
}
if tlsConfig.SSLPolicyRef != nil && tlsConfig.SSLPolicyRef.SSLPolicyId > 0 {
err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, tlsConfig.SSLPolicyRef.SSLPolicyId, userId)
err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, userId, tlsConfig.SSLPolicyRef.SSLPolicyId)
if err != nil {
return nil, err
}

View File

@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"sort"
"time"
)
@@ -90,10 +91,13 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
}
// 规则分组
groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, userId, req.ServerId, monthFrom, monthTo, 0, 10)
groupStats, err := stats.SharedServerHTTPFirewallDailyStatDAO.GroupDailyCount(tx, userId, req.ServerId, monthFrom, monthTo, 0, 20)
if err != nil {
return nil, err
}
// 合并同名
var groupNamedStatsMap = map[string]*stats.ServerHTTPFirewallDailyStat{} // name => *ServerHTTPFirewallDailyStat
for _, stat := range groupStats {
ruleGroupName, err := models.SharedHTTPFirewallRuleGroupDAO.FindHTTPFirewallRuleGroupName(tx, int64(stat.HttpFirewallRuleGroupId))
if err != nil {
@@ -103,11 +107,26 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
continue
}
namedStat, ok := groupNamedStatsMap[ruleGroupName]
if ok {
namedStat.Count += stat.Count
} else {
groupNamedStatsMap[ruleGroupName] = stat
}
}
for ruleGroupName, stat := range groupNamedStatsMap {
resp.HttpFirewallRuleGroups = append(resp.HttpFirewallRuleGroups, &pb.ComposeServerHTTPFirewallDashboardResponse_HTTPFirewallRuleGroupStat{
HttpFirewallRuleGroup: &pb.HTTPFirewallRuleGroup{Id: int64(stat.HttpFirewallRuleGroupId), Name: ruleGroupName},
Count: int64(stat.Count),
})
}
sort.Slice(resp.HttpFirewallRuleGroups, func(i, j int) bool {
return resp.HttpFirewallRuleGroups[i].Count > resp.HttpFirewallRuleGroups[j].Count
})
if len(resp.HttpFirewallRuleGroups) > 10 {
resp.HttpFirewallRuleGroups = resp.HttpFirewallRuleGroups[:10]
}
// 每日趋势
dayBefore := timeutil.Format("Ymd", date.AddDate(0, 0, -14))
@@ -116,7 +135,7 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
return nil, err
}
{
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "log", dayBefore, day)
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, []string{"log", "tag"}, dayBefore, day)
if err != nil {
return nil, err
}
@@ -129,7 +148,7 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
}
}
{
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "block", dayBefore, day)
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, []string{"block", "page"}, dayBefore, day)
if err != nil {
return nil, err
}
@@ -142,7 +161,7 @@ func (this *ServerHTTPFirewallDailyStatService) ComposeServerHTTPFirewallDashboa
}
}
{
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, "captcha", dayBefore, day)
statList, err := stats.SharedServerHTTPFirewallDailyStatDAO.FindDailyStats(tx, userId, req.ServerId, []string{"captcha"}, dayBefore, day)
if err != nil {
return nil, err
}

View File

@@ -9,12 +9,11 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models/regions"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/rpc/tasks"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
@@ -208,11 +207,12 @@ func (this *ServerStatBoardService) ComposeServerStatNodeClusterBoard(ctx contex
})
}
charts, err := this.findNodeClusterMetricDataCharts(tx, req.NodeClusterId, 0, 0, serverconfigs.MetricItemCategoryHTTP)
if err != nil {
return nil, err
var pbCharts []*pb.MetricDataChart
charts, ok := tasks.SharedCacheTaskManager.GetCluster(tasks.CacheKeyFindNodeClusterMetricDataCharts, req.NodeClusterId, serverconfigs.MetricItemCategoryHTTP)
if ok {
pbCharts = charts.([]*pb.MetricDataChart)
}
result.MetricDataCharts = charts
result.MetricDataCharts = pbCharts
return result, nil
}
@@ -443,11 +443,12 @@ func (this *ServerStatBoardService) ComposeServerStatNodeBoard(ctx context.Conte
// 指标
var clusterId = int64(node.ClusterId)
charts, err := this.findNodeClusterMetricDataCharts(tx, clusterId, req.NodeId, 0, serverconfigs.MetricItemCategoryHTTP)
if err != nil {
return nil, err
var pbCharts []*pb.MetricDataChart
charts, ok := tasks.SharedCacheTaskManager.GetNode(tasks.CacheKeyFindNodeMetricDataCharts, clusterId, req.NodeId, serverconfigs.MetricItemCategoryHTTP)
if ok {
pbCharts = charts.([]*pb.MetricDataChart)
}
result.MetricDataCharts = charts
result.MetricDataCharts = pbCharts
return result, nil
}
@@ -560,182 +561,13 @@ func (this *ServerStatBoardService) ComposeServerStatBoard(ctx context.Context,
case serverconfigs.ServerTypeUDPProxy:
metricCategory = serverconfigs.MetricItemCategoryUDP
}
charts, err := this.findNodeClusterMetricDataCharts(tx, clusterId, 0, req.ServerId, metricCategory)
if err != nil {
return nil, err
var pbCharts []*pb.MetricDataChart
charts, ok := tasks.SharedCacheTaskManager.GetServer(tasks.CacheKeyFindServerMetricDataCharts, clusterId, req.ServerId, metricCategory)
if ok {
pbCharts = charts.([]*pb.MetricDataChart)
}
result.MetricDataCharts = charts
result.MetricDataCharts = pbCharts
return result, nil
}
// 查找集群、节点和服务的指标数据
func (this *ServerStatBoardService) findNodeClusterMetricDataCharts(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, category string) (result []*pb.MetricDataChart, err error) {
// 集群指标
clusterMetricItems, err := models.SharedNodeClusterMetricItemDAO.FindAllClusterItems(tx, clusterId, category)
if err != nil {
return nil, err
}
var pbMetricCharts = []*pb.MetricDataChart{}
var metricItemIds = []int64{}
var items = []*models.MetricItem{}
for _, clusterMetricItem := range clusterMetricItems {
if clusterMetricItem.IsOn != 1 {
continue
}
var itemId = int64(clusterMetricItem.ItemId)
item, err := models.SharedMetricItemDAO.FindEnabledMetricItem(tx, itemId)
if err != nil {
return nil, err
}
if item == nil || item.IsOn == 0 {
continue
}
items = append(items, item)
metricItemIds = append(metricItemIds, itemId)
}
publicMetricItems, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, category, nil)
if err != nil {
return nil, err
}
for _, item := range publicMetricItems {
if item.IsOn != 1 {
continue
}
if lists.ContainsInt64(metricItemIds, int64(item.Id)) {
continue
}
items = append(items, item)
}
for _, item := range items {
var itemId = int64(item.Id)
charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
if err != nil {
return nil, err
}
for _, chart := range charts {
if chart.IsOn == 0 {
continue
}
var pbChart = &pb.MetricChart{
Id: int64(chart.Id),
Name: chart.Name,
Type: chart.Type,
WidthDiv: chart.WidthDiv,
ParamsJSON: nil,
IsOn: chart.IsOn == 1,
MaxItems: types.Int32(chart.MaxItems),
MetricItem: &pb.MetricItem{
Id: itemId,
PeriodUnit: item.PeriodUnit,
Period: types.Int32(item.Period),
Name: item.Name,
Value: item.Value,
Category: item.Category,
Keys: item.DecodeKeys(),
Code: item.Code,
IsOn: item.IsOn == 1,
},
}
var pbStats = []*pb.MetricStat{}
switch chart.Type {
case serverconfigs.MetricChartTypeTimeLine:
var itemStats []*models.MetricStat
if serverId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithServerId(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else if nodeId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithNodeId(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else {
itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithClusterId(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
}
if err != nil {
return nil, err
}
for _, stat := range itemStats {
// 当前时间总和
var count int64
var total float32
if serverId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
} else if nodeId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
} else {
count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
}
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
default:
var itemStats []*models.MetricStat
if serverId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithServerIdAndLastTime(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else if nodeId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithNodeIdAndLastTime(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else {
itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithClusterIdAndLastTime(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
}
if err != nil {
return nil, err
}
for _, stat := range itemStats {
// 当前时间总和
var count int64
var total float32
if serverId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
} else if nodeId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
} else {
count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
}
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
}
pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
MetricChart: pbChart,
MetricStats: pbStats,
})
}
}
return pbMetricCharts, nil
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models/acme"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/sslconfigs"
"github.com/iwind/TeaGo/types"
)
// SSLCertService SSL证书相关服务
@@ -180,3 +181,132 @@ func (this *SSLCertService) ListSSLCerts(ctx context.Context, req *pb.ListSSLCer
}
return &pb.ListSSLCertsResponse{SslCertsJSON: certConfigsJSON}, nil
}
// CountAllSSLCertsWithOCSPError 计算有OCSP错误的证书数量
func (this *SSLCertService) CountAllSSLCertsWithOCSPError(ctx context.Context, req *pb.CountAllSSLCertsWithOCSPErrorRequest) (*pb.RPCCountResponse, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
count, err := models.SharedSSLCertDAO.CountAllSSLCertsWithOCSPError(tx, req.Keyword)
if err != nil {
return nil, err
}
return this.SuccessCount(count)
}
// ListSSLCertsWithOCSPError 列出有OCSP错误的证书
func (this *SSLCertService) ListSSLCertsWithOCSPError(ctx context.Context, req *pb.ListSSLCertsWithOCSPErrorRequest) (*pb.ListSSLCertsWithOCSPErrorResponse, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
certs, err := models.SharedSSLCertDAO.ListSSLCertsWithOCSPError(tx, req.Keyword, req.Offset, req.Size)
if err != nil {
return nil, err
}
var pbCerts = []*pb.SSLCert{}
for _, cert := range certs {
pbCerts = append(pbCerts, &pb.SSLCert{
Id: int64(cert.Id),
IsOn: cert.IsOn == 1,
Name: cert.Name,
TimeBeginAt: types.Int64(cert.TimeBeginAt),
TimeEndAt: types.Int64(cert.TimeEndAt),
DnsNames: cert.DecodeDNSNames(),
CommonNames: cert.DecodeCommonNames(),
IsACME: cert.IsACME == 1,
AcmeTaskId: int64(cert.AcmeTaskId),
Ocsp: []byte(cert.Ocsp),
OcspIsUpdated: cert.OcspIsUpdated == 1,
OcspError: cert.OcspError,
Description: cert.Description,
IsCA: cert.IsCA == 1,
ServerName: cert.ServerName,
CreatedAt: int64(cert.CreatedAt),
UpdatedAt: int64(cert.UpdatedAt),
})
}
return &pb.ListSSLCertsWithOCSPErrorResponse{
SslCerts: pbCerts,
}, nil
}
// IgnoreSSLCertsWithOCSPError 忽略一组OCSP证书错误
func (this *SSLCertService) IgnoreSSLCertsWithOCSPError(ctx context.Context, req *pb.IgnoreSSLCertsWithOCSPErrorRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedSSLCertDAO.IgnoreSSLCertsWithOCSPError(tx, req.SslCertIds)
if err != nil {
return nil, err
}
return this.Success()
}
// ResetSSLCertsWithOCSPError 重置一组证书OCSP错误状态
func (this *SSLCertService) ResetSSLCertsWithOCSPError(ctx context.Context, req *pb.ResetSSLCertsWithOCSPErrorRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedSSLCertDAO.ResetSSLCertsWithOCSPError(tx, req.SslCertIds)
if err != nil {
return nil, err
}
return this.Success()
}
// ResetAllSSLCertsWithOCSPError 重置所有证书OCSP错误状态
func (this *SSLCertService) ResetAllSSLCertsWithOCSPError(ctx context.Context, req *pb.ResetAllSSLCertsWithOCSPErrorRequest) (*pb.RPCSuccess, error) {
_, err := this.ValidateAdmin(ctx, 0)
if err != nil {
return nil, err
}
var tx = this.NullTx()
err = models.SharedSSLCertDAO.ResetAllSSLCertsWithOCSPError(tx)
if err != nil {
return nil, err
}
return this.Success()
}
// ListUpdatedSSLCertOCSP 读取证书的OCSP
func (this *SSLCertService) ListUpdatedSSLCertOCSP(ctx context.Context, req *pb.ListUpdatedSSLCertOCSPRequest) (*pb.ListUpdatedSSLCertOCSPResponse, error) {
_, err := this.ValidateNode(ctx)
if err != nil {
return nil, err
}
var tx = this.NullTx()
certs, err := models.SharedSSLCertDAO.ListCertOCSPAfterVersion(tx, req.Version, int64(req.Size))
if err != nil {
return nil, err
}
var result = []*pb.ListUpdatedSSLCertOCSPResponse_SSLCertOCSP{}
for _, cert := range certs {
result = append(result, &pb.ListUpdatedSSLCertOCSPResponse_SSLCertOCSP{
SslCertId: int64(cert.Id),
Data: []byte(cert.Ocsp),
ExpiresAt: int64(cert.OcspExpiresAt),
Version: int64(cert.OcspUpdatedVersion),
})
}
return &pb.ListUpdatedSSLCertOCSPResponse{
SslCertOCSP: result,
}, nil
}

View File

@@ -3,6 +3,7 @@ package services
import (
"context"
"encoding/json"
"errors"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
rpcutils "github.com/TeaOSLab/EdgeAPI/internal/rpc/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
@@ -43,7 +44,7 @@ func (this *SSLPolicyService) CreateSSLPolicy(ctx context.Context, req *pb.Creat
// TODO
}
policyId, err := models.SharedSSLPolicyDAO.CreatePolicy(tx, adminId, userId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
policyId, err := models.SharedSSLPolicyDAO.CreatePolicy(tx, adminId, userId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.OcspIsOn, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
if err != nil {
return nil, err
}
@@ -62,13 +63,13 @@ func (this *SSLPolicyService) UpdateSSLPolicy(ctx context.Context, req *pb.Updat
tx := this.NullTx()
if userId > 0 {
err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, req.SslPolicyId, userId)
err := models.SharedSSLPolicyDAO.CheckUserPolicy(tx, userId, req.SslPolicyId)
if err != nil {
return nil, err
return nil, errors.New("check ssl policy failed: " + err.Error())
}
}
err = models.SharedSSLPolicyDAO.UpdatePolicy(tx, req.SslPolicyId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
err = models.SharedSSLPolicyDAO.UpdatePolicy(tx, req.SslPolicyId, req.Http2Enabled, req.MinVersion, req.SslCertsJSON, req.HstsJSON, req.OcspIsOn, req.ClientAuthType, req.ClientCACertsJSON, req.CipherSuitesIsOn, req.CipherSuites)
if err != nil {
return nil, err
}

View File

@@ -0,0 +1,494 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package tasks
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/stats"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"sync"
"time"
)
var SharedCacheTaskManager = NewCacheTaskManager()
const (
// global keys
CacheKeyFindAllMetricDataCharts = "findAllMetricDataCharts"
CacheKeyFindGlobalTopDomains = "globalFindTopDomains"
// cluster keys
CacheKeyFindNodeClusterMetricDataCharts = "findNodeClusterMetricDataCharts"
// node keys
CacheKeyFindNodeMetricDataCharts = "findNodeMetricDataCharts"
// server keys
CacheKeyFindServerMetricDataCharts = "findServerMetricDataCharts"
)
func init() {
dbs.OnReadyDone(func() {
goman.New(func() {
_ = SharedCacheTaskManager.Loop()
SharedCacheTaskManager.Start()
})
})
}
type CacheTaskManager struct {
cacheMap map[string]interface{}
locker sync.Mutex
nodeTasksTooSlow bool // 记录节点任务是否太慢,如果太慢就不再后台执行
ticker *time.Ticker
}
func NewCacheTaskManager() *CacheTaskManager {
return &CacheTaskManager{
cacheMap: map[string]interface{}{},
ticker: time.NewTicker(5 * time.Minute),
}
}
func (this *CacheTaskManager) Start() {
for range this.ticker.C {
err := this.Loop()
if err != nil {
remotelogs.Error("CACHE_TASK_MANAGER", err.Error())
}
}
}
func (this *CacheTaskManager) Loop() error {
var tx *dbs.Tx
// Admin看板指标数据
{
value, err := this.findAllMetricDataCharts(tx)
if err != nil {
return err
}
this.locker.Lock()
this.cacheMap[CacheKeyFindAllMetricDataCharts] = value
this.locker.Unlock()
}
{
var hourFrom = timeutil.Format("YmdH", time.Now().Add(-23*time.Hour))
var hourTo = timeutil.Format("YmdH")
value, err := stats.SharedServerDomainHourlyStatDAO.FindTopDomainStats(tx, hourFrom, hourTo, 10)
if err != nil {
return err
}
var composedKey = CacheKeyFindGlobalTopDomains
this.locker.Lock()
this.cacheMap[composedKey] = value
this.locker.Unlock()
}
// 集群数据
clusterIds, err := models.SharedNodeClusterDAO.FindAllEnabledNodeClusterIds(tx)
if err != nil {
return err
}
for _, clusterId := range clusterIds {
// metric charts
{
var category = serverconfigs.MetricItemCategoryHTTP
var composedKey = CacheKeyFindNodeClusterMetricDataCharts + "@" + types.String(clusterId) + "@" + category
value, err := this.findNodeClusterMetricDataCharts(tx, clusterId, 0, 0, category)
if err != nil {
return err
}
this.locker.Lock()
this.cacheMap[composedKey] = value
this.locker.Unlock()
}
// nodes
var before = time.Now()
if !this.nodeTasksTooSlow {
nodeIds, err := models.SharedNodeDAO.FindEnabledNodeIdsWithClusterId(tx, clusterId)
if err != nil {
return err
}
for _, nodeId := range nodeIds {
// metric
{
var category = serverconfigs.MetricItemCategoryHTTP
var composedKey = CacheKeyFindNodeMetricDataCharts + "@" + types.String(clusterId) + "@" + types.String(nodeId) + "@" + category
value, err := this.findNodeClusterMetricDataCharts(tx, clusterId, nodeId, 0, category)
if err != nil {
return err
}
this.locker.Lock()
this.cacheMap[composedKey] = value
this.locker.Unlock()
}
}
}
var cost = time.Since(before).Seconds()
if cost > 600 {
this.nodeTasksTooSlow = true
}
}
return nil
}
func (this *CacheTaskManager) Get(key string) (value interface{}, ok bool) {
this.locker.Lock()
defer this.locker.Unlock()
if key == CacheKeyFindAllMetricDataCharts {
value, ok = this.cacheMap[key]
return
}
return
}
func (this *CacheTaskManager) GetCluster(key string, clusterId int64, category string) (value interface{}, ok bool) {
this.locker.Lock()
defer this.locker.Unlock()
var composedKey = key + "@" + types.String(clusterId) + "@" + category
value, ok = this.cacheMap[composedKey]
return
}
func (this *CacheTaskManager) GetNode(key string, clusterId int64, nodeId int64, category string) (value interface{}, ok bool) {
this.locker.Lock()
var composedKey = key + "@" + types.String(clusterId) + "@" + types.String(nodeId) + "@" + category
value, ok = this.cacheMap[composedKey]
this.locker.Unlock()
if ok {
result, err := this.findNodeClusterMetricDataCharts(nil, clusterId, nodeId, 0, category)
if err == nil {
value = result
}
}
return
}
func (this *CacheTaskManager) GetServer(key string, clusterId int64, serverId int64, category string) (value interface{}, ok bool) {
switch key {
case CacheKeyFindServerMetricDataCharts:
var tx *dbs.Tx
v, err := this.findNodeClusterMetricDataCharts(tx, clusterId, 0, serverId, category)
if err != nil {
return nil, false
}
return v, true
}
return
}
func (this *CacheTaskManager) GetGlobalTopDomains() (value interface{}, ok bool) {
this.locker.Lock()
defer this.locker.Unlock()
var composedKey = CacheKeyFindGlobalTopDomains
value, ok = this.cacheMap[composedKey]
return
}
// 所有集群的指标统计
func (this *CacheTaskManager) findAllMetricDataCharts(tx *dbs.Tx) (result []*pb.MetricDataChart, err error) {
// 集群指标
items, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, serverconfigs.MetricItemCategoryHTTP, nil)
if err != nil {
return nil, err
}
var pbMetricCharts = []*pb.MetricDataChart{}
for _, item := range items {
var itemId = int64(item.Id)
charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
if err != nil {
return nil, err
}
for _, chart := range charts {
if chart.IsOn == 0 {
continue
}
var pbChart = &pb.MetricChart{
Id: int64(chart.Id),
Name: chart.Name,
Type: chart.Type,
WidthDiv: chart.WidthDiv,
ParamsJSON: nil,
IsOn: chart.IsOn == 1,
MaxItems: types.Int32(chart.MaxItems),
MetricItem: &pb.MetricItem{
Id: itemId,
PeriodUnit: item.PeriodUnit,
Period: types.Int32(item.Period),
Name: item.Name,
Value: item.Value,
Category: item.Category,
Keys: item.DecodeKeys(),
Code: item.Code,
IsOn: item.IsOn == 1,
},
}
var pbStats = []*pb.MetricStat{}
switch chart.Type {
case serverconfigs.MetricChartTypeTimeLine:
itemStats, err := models.SharedMetricStatDAO.FindLatestItemStats(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
if err != nil {
return nil, err
}
for _, stat := range itemStats {
// 当前时间总和
count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
default:
itemStats, err := models.SharedMetricStatDAO.FindItemStatsAtLastTime(tx, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
if err != nil {
return nil, err
}
for _, stat := range itemStats {
count, total, err := models.SharedMetricSumStatDAO.FindSumAtTime(tx, stat.Time, itemId, types.Int32(item.Version))
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
}
pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
MetricChart: pbChart,
MetricStats: pbStats,
})
}
}
return pbMetricCharts, nil
}
// 某个集群、节点或者服务的指标统计
func (this *CacheTaskManager) findNodeClusterMetricDataCharts(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, category string) (result []*pb.MetricDataChart, err error) {
// 集群指标
clusterMetricItems, err := models.SharedNodeClusterMetricItemDAO.FindAllClusterItems(tx, clusterId, category)
if err != nil {
return nil, err
}
var pbMetricCharts = []*pb.MetricDataChart{}
var metricItemIds = []int64{}
var items = []*models.MetricItem{}
for _, clusterMetricItem := range clusterMetricItems {
if clusterMetricItem.IsOn != 1 {
continue
}
var itemId = int64(clusterMetricItem.ItemId)
item, err := models.SharedMetricItemDAO.FindEnabledMetricItem(tx, itemId)
if err != nil {
return nil, err
}
if item == nil || item.IsOn == 0 {
continue
}
items = append(items, item)
metricItemIds = append(metricItemIds, itemId)
}
publicMetricItems, err := models.SharedMetricItemDAO.FindAllPublicItems(tx, category, nil)
if err != nil {
return nil, err
}
for _, item := range publicMetricItems {
if item.IsOn != 1 {
continue
}
if lists.ContainsInt64(metricItemIds, int64(item.Id)) {
continue
}
items = append(items, item)
}
for _, item := range items {
var itemId = int64(item.Id)
charts, err := models.SharedMetricChartDAO.FindAllEnabledCharts(tx, itemId)
if err != nil {
return nil, err
}
for _, chart := range charts {
if chart.IsOn == 0 {
continue
}
var pbChart = &pb.MetricChart{
Id: int64(chart.Id),
Name: chart.Name,
Type: chart.Type,
WidthDiv: chart.WidthDiv,
ParamsJSON: nil,
IsOn: chart.IsOn == 1,
MaxItems: types.Int32(chart.MaxItems),
MetricItem: &pb.MetricItem{
Id: itemId,
PeriodUnit: item.PeriodUnit,
Period: types.Int32(item.Period),
Name: item.Name,
Value: item.Value,
Category: item.Category,
Keys: item.DecodeKeys(),
Code: item.Code,
IsOn: item.IsOn == 1,
},
}
var pbStats = []*pb.MetricStat{}
switch chart.Type {
case serverconfigs.MetricChartTypeTimeLine:
var itemStats []*models.MetricStat
if serverId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithServerId(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else if nodeId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithNodeId(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else {
itemStats, err = models.SharedMetricStatDAO.FindLatestItemStatsWithClusterId(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
}
if err != nil {
return nil, err
}
for _, stat := range itemStats {
// 当前时间总和
var count int64
var total float32
if serverId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
} else if nodeId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
} else {
count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
}
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
default:
var itemStats []*models.MetricStat
if serverId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithServerIdAndLastTime(tx, serverId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else if nodeId > 0 {
itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithNodeIdAndLastTime(tx, nodeId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
} else {
itemStats, err = models.SharedMetricStatDAO.FindItemStatsWithClusterIdAndLastTime(tx, clusterId, itemId, chart.IgnoreEmptyKeys == 1, chart.DecodeIgnoredKeys(), types.Int32(item.Version), 10)
}
if err != nil {
return nil, err
}
for _, stat := range itemStats {
// 当前时间总和
var count int64
var total float32
if serverId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindServerSum(tx, serverId, stat.Time, itemId, types.Int32(item.Version))
} else if nodeId > 0 {
count, total, err = models.SharedMetricSumStatDAO.FindNodeSum(tx, nodeId, stat.Time, itemId, types.Int32(item.Version))
} else {
count, total, err = models.SharedMetricSumStatDAO.FindClusterSum(tx, clusterId, stat.Time, itemId, types.Int32(item.Version))
}
if err != nil {
return nil, err
}
pbStats = append(pbStats, &pb.MetricStat{
Id: int64(stat.Id),
Hash: stat.Hash,
ServerId: 0,
ItemId: 0,
Keys: stat.DecodeKeys(),
Value: types.Float32(stat.Value),
Time: stat.Time,
Version: 0,
NodeCluster: nil,
Node: nil,
Server: nil,
SumCount: count,
SumTotal: total,
})
}
}
pbMetricCharts = append(pbMetricCharts, &pb.MetricDataChart{
MetricChart: pbChart,
MetricStats: pbStats,
})
}
}
return pbMetricCharts, nil
}

View File

@@ -6,11 +6,11 @@ import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/go-yaml/yaml"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/cmd"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/types"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
"strconv"

File diff suppressed because one or more lines are too long

View File

@@ -45,7 +45,10 @@ func (this *SQLDump) Dump(db *dbs.DB) (result *SQLDumpResult, err error) {
}
for _, tableName := range tableNames {
// 忽略一些分表
if strings.HasPrefix(tableName, "edgeHTTPAccessLogs_") {
if strings.HasPrefix(strings.ToLower(tableName), strings.ToLower("edgeHTTPAccessLogs_")) {
continue
}
if strings.HasPrefix(strings.ToLower(tableName), strings.ToLower("edgeNSAccessLogs_")) {
continue
}

View File

@@ -2,20 +2,19 @@ package setup
import (
"encoding/json"
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/go-yaml/yaml"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"gopkg.in/yaml.v3"
"io/ioutil"
"time"
)
@@ -113,7 +112,7 @@ func (this *SQLExecutor) checkData(db *dbs.DB) error {
}
// 更新版本号
err = this.updateVersion(db, teaconst.Version)
err = this.updateVersion(db, ComposeSQLVersion())
if err != nil {
return err
}

View File

@@ -69,6 +69,9 @@ var upgradeFuncs = []*upgradeVersion{
{
"0.4.1", upgradeV0_4_1,
},
{
"0.4.5", upgradeV0_4_5,
},
}
// UpgradeSQLData 升级SQL数据
@@ -590,3 +593,48 @@ func upgradeV0_4_1(db *dbs.DB) error {
return nil
}
// v0.4.5
func upgradeV0_4_5(db *dbs.DB) error {
// 升级访问日志自动分表
{
var dao = models.NewSysSettingDAO()
valueJSON, err := dao.ReadSetting(nil, systemconfigs.SettingCodeAccessLogQueue)
if err != nil {
return err
}
if len(valueJSON) > 0 {
var config = &serverconfigs.AccessLogQueueConfig{}
err = json.Unmarshal(valueJSON, config)
if err == nil && config.RowsPerTable == 0 {
config.EnableAutoPartial = true
config.RowsPerTable = 500_000
configJSON, err := json.Marshal(config)
if err == nil {
err = dao.UpdateSetting(nil, systemconfigs.SettingCodeAccessLogQueue, configJSON)
if err != nil {
return err
}
}
}
}
}
// 升级一个SQL注入规则
{
var dao = models.NewHTTPFirewallRuleDAO()
ones, _, err := dao.Instance.FindOnes(`SELECT id FROM edgeHTTPFirewallRules WHERE value=?`, "(updatexml|extractvalue|ascii|ord|char|chr|count|concat|rand|floor|substr|length|len|user|database|benchmark|analyse)\\s*\\(")
if err != nil {
return err
}
for _, one := range ones {
var ruleId = one.GetInt64("id")
_, err = dao.Instance.Exec(`UPDATE edgeHTTPFirewallRules SET value=? WHERE id=? LIMIT 1`, `\b(updatexml|extractvalue|ascii|ord|char|chr|count|concat|rand|floor|substr|length|len|user|database|benchmark|analyse)\s*\(.*\)`, ruleId)
if err != nil {
return err
}
}
}
return nil
}

View File

@@ -116,3 +116,20 @@ func TestUpgradeSQLData_v0_4_1(t *testing.T) {
}
t.Log("ok")
}
func TestUpgradeSQLData_v0_4_5(t *testing.T) {
db, err := dbs.NewInstanceFromConfig(&dbs.DBConfig{
Driver: "mysql",
Dsn: "root:123456@tcp(127.0.0.1:3306)/db_edge?charset=utf8mb4&timeout=30s",
Prefix: "edge",
})
if err != nil {
t.Fatal(err)
}
err = upgradeV0_4_5(db)
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

20
internal/setup/utils.go Normal file
View File

@@ -0,0 +1,20 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package setup
import (
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"strings"
)
func ComposeSQLVersion() string {
var version = teaconst.Version
if len(teaconst.SQLVersion) == 0 {
return version
}
if strings.Count(version, ".") <= 2 {
return version + "." + teaconst.SQLVersion
}
return version
}

View File

@@ -0,0 +1,12 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package setup_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/setup"
"testing"
)
func TestComposeSQLVersion(t *testing.T) {
t.Log(setup.ComposeSQLVersion())
}

View File

@@ -38,34 +38,17 @@ func (this *LogTask) Run() {
}
func (this *LogTask) runClean() {
ticker := utils.NewTicker(24 * time.Hour)
var ticker = utils.NewTicker(24 * time.Hour)
for ticker.Wait() {
err := this.loopClean(86400)
err := this.loopClean()
if err != nil {
logs.Println("[TASK][LOG]" + err.Error())
}
}
}
func (this *LogTask) loopClean(seconds int64) error {
// 检查上次运行时间,防止重复运行
settingKey := "logTaskCleanLoop"
timestamp := time.Now().Unix()
c, err := models.SharedSysSettingDAO.CompareInt64Setting(nil, settingKey, timestamp-seconds)
if err != nil {
return err
}
if c > 0 {
return nil
}
// 记录时间
err = models.SharedSysSettingDAO.UpdateSetting(nil, settingKey, []byte(numberutils.FormatInt64(timestamp)))
if err != nil {
return err
}
configKey := "adminLogConfig"
func (this *LogTask) loopClean() error {
var configKey = "adminLogConfig"
valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, configKey)
if err != nil {
return err
@@ -74,7 +57,7 @@ func (this *LogTask) loopClean(seconds int64) error {
return nil
}
config := &systemconfigs.LogConfig{}
var config = &systemconfigs.LogConfig{}
err = json.Unmarshal(valueJSON, config)
if err != nil {
return err
@@ -89,7 +72,7 @@ func (this *LogTask) loopClean(seconds int64) error {
}
func (this *LogTask) runMonitor() {
ticker := utils.NewTicker(1 * time.Minute)
var ticker = utils.NewTicker(1 * time.Minute)
for ticker.Wait() {
err := this.loopMonitor(60)
if err != nil {
@@ -100,8 +83,8 @@ func (this *LogTask) runMonitor() {
func (this *LogTask) loopMonitor(seconds int64) error {
// 检查上次运行时间,防止重复运行
settingKey := "logTaskMonitorLoop"
timestamp := time.Now().Unix()
var settingKey = "logTaskMonitorLoop"
var timestamp = time.Now().Unix()
c, err := models.SharedSysSettingDAO.CompareInt64Setting(nil, settingKey, timestamp-seconds)
if err != nil {
return err
@@ -116,7 +99,7 @@ func (this *LogTask) loopMonitor(seconds int64) error {
return err
}
configKey := "adminLogConfig"
var configKey = "adminLogConfig"
valueJSON, err := models.SharedSysSettingDAO.ReadSetting(nil, configKey)
if err != nil {
return err
@@ -125,7 +108,7 @@ func (this *LogTask) loopMonitor(seconds int64) error {
return nil
}
config := &systemconfigs.LogConfig{}
var config = &systemconfigs.LogConfig{}
err = json.Unmarshal(valueJSON, config)
if err != nil {
return err

View File

@@ -9,7 +9,7 @@ func TestLogTask_loopClean(t *testing.T) {
dbs.NotifyReady()
task := NewLogTask()
err := task.loopClean(5)
err := task.loopClean()
if err != nil {
t.Fatal(err)
}

View File

@@ -103,23 +103,18 @@ func (this *ServerAccessLogCleaner) cleanDB(db *dbs.DB, endDay string) error {
return errors.New("invalid column names: " + strings.Join(columnNames, ", "))
}
columnName := columnNames[0]
var reg = regexp.MustCompile(`^(?i)(edgeHTTPAccessLogs|edgeNSAccessLogs)_(\d{8})(_\d{4})?$`)
for _, one := range ones {
tableName := one.GetString(columnName)
var tableName = one.GetString(columnName)
if len(tableName) == 0 {
continue
}
ok, err := regexp.MatchString(`^(?i)(edgeHTTPAccessLogs|edgeNSAccessLogs)_(\d{8})$`, tableName)
if err != nil {
return err
}
if !ok {
if !reg.MatchString(tableName) {
continue
}
index := strings.LastIndex(tableName, "_")
if index < 0 {
continue
}
day := tableName[index+1:]
var matches = reg.FindStringSubmatch(tableName)
var day = matches[2]
if day < endDay {
_, err = db.Exec("DROP TABLE " + tableName)
if err != nil {

View File

@@ -1,6 +1,7 @@
package tasks
package tasks_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/tasks"
"github.com/iwind/TeaGo/dbs"
"testing"
)
@@ -8,7 +9,7 @@ import (
func TestServerAccessLogCleaner_Loop(t *testing.T) {
dbs.NotifyReady()
task := NewServerAccessLogCleaner()
task := tasks.NewServerAccessLogCleaner()
err := task.Loop()
if err != nil {
t.Fatal(err)

View File

@@ -0,0 +1,199 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package tasks
import (
"bytes"
"crypto"
"crypto/tls"
"crypto/x509"
"errors"
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/goman"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/iwind/TeaGo/dbs"
"golang.org/x/crypto/ocsp"
"io/ioutil"
"net/http"
"time"
)
func init() {
dbs.OnReadyDone(func() {
goman.New(func() {
NewSSLCertUpdateOCSPTask().Start()
})
})
}
type SSLCertUpdateOCSPTask struct {
ticker *time.Ticker
httpClient *http.Client
}
func NewSSLCertUpdateOCSPTask() *SSLCertUpdateOCSPTask {
return &SSLCertUpdateOCSPTask{
ticker: time.NewTicker(1 * time.Minute),
httpClient: utils.SharedHttpClient(5 * time.Second),
}
}
func (this *SSLCertUpdateOCSPTask) Start() {
for range this.ticker.C {
err := this.Loop(true)
if err != nil {
remotelogs.Error("SSLCertUpdateOCSPTask", err.Error())
}
}
}
func (this *SSLCertUpdateOCSPTask) Loop(checkLock bool) error {
if checkLock {
ok, err := models.SharedSysLockerDAO.Lock(nil, "ssl_cert_update_ocsp_task", 60-1) // 假设执行时间为1秒
if err != nil {
return err
}
if !ok {
return nil
}
}
var tx *dbs.Tx
// TODO 将来可以设置单次任务条数
var size int64 = 60
var maxTries = 5
certs, err := models.SharedSSLCertDAO.ListCertsToUpdateOCSP(tx, maxTries, size)
if err != nil {
return errors.New("list certs failed: " + err.Error())
}
if len(certs) == 0 {
return nil
}
// 锁定
for _, cert := range certs {
err := models.SharedSSLCertDAO.PrepareCertOCSPUpdating(tx, int64(cert.Id))
if err != nil {
return errors.New("prepare cert ocsp updating failed: " + err.Error())
}
}
for _, cert := range certs {
ocspData, expiresAt, err := this.UpdateCertOCSP(cert)
var errString = ""
var hasErr = false
if err != nil {
errString = err.Error()
hasErr = true
remotelogs.Warn("SSLCertUpdateOCSPTask", "update ocsp failed: "+errString)
}
err = models.SharedSSLCertDAO.UpdateCertOCSP(tx, int64(cert.Id), ocspData, expiresAt, hasErr, errString)
if err != nil {
return errors.New("update ocsp failed: " + err.Error())
}
}
return nil
}
// UpdateCertOCSP 更新单个证书OCSP
func (this *SSLCertUpdateOCSPTask) UpdateCertOCSP(certOne *models.SSLCert) (ocspData []byte, expiresAt int64, err error) {
if certOne.IsCA == 1 || len(certOne.CertData) == 0 || len(certOne.KeyData) == 0 {
return
}
keyPair, err := tls.X509KeyPair([]byte(certOne.CertData), []byte(certOne.KeyData))
if err != nil {
return nil, 0, errors.New("parse certificate failed: " + err.Error())
}
if len(keyPair.Certificate) == 0 {
return nil, 0, nil
}
var certData = keyPair.Certificate[0]
cert, err := x509.ParseCertificate(certData)
if err != nil {
return nil, 0, errors.New("parse certificate block failed: " + err.Error())
}
// 是否已过期
var now = time.Now()
if cert.NotBefore.After(now) || cert.NotAfter.Before(now) {
return nil, 0, nil
}
if len(cert.IssuingCertificateURL) == 0 || len(cert.OCSPServer) == 0 {
return nil, 0, nil
}
if len(cert.DNSNames) == 0 {
return nil, 0, nil
}
var issuerURL = cert.IssuingCertificateURL[0]
var ocspServerURL = cert.OCSPServer[0]
issuerReq, err := http.NewRequest(http.MethodGet, issuerURL, nil)
if err != nil {
return nil, 0, errors.New("request issuer certificate failed: " + err.Error())
}
issuerReq.Header.Set("User-Agent", teaconst.ProductName+"/"+teaconst.Version)
issuerResp, err := this.httpClient.Do(issuerReq)
if err != nil {
return nil, 0, errors.New("request issuer certificate failed: '" + issuerURL + "': " + err.Error())
}
defer func() {
_ = issuerResp.Body.Close()
}()
issuerData, err := ioutil.ReadAll(issuerResp.Body)
if err != nil {
return nil, 0, errors.New("read issuer certificate failed: '" + issuerURL + "': " + err.Error())
}
issuerCert, err := x509.ParseCertificate(issuerData)
if err != nil {
return nil, 0, errors.New("parse issuer certificate failed: '" + issuerURL + "': " + err.Error())
}
buf, err := ocsp.CreateRequest(cert, issuerCert, &ocsp.RequestOptions{
Hash: crypto.SHA1,
})
if err != nil {
return nil, 0, errors.New("create ocsp request failed: " + err.Error())
}
ocspReq, err := http.NewRequest(http.MethodPost, ocspServerURL, bytes.NewBuffer(buf))
if err != nil {
return nil, 0, errors.New("request ocsp failed: " + err.Error())
}
ocspReq.Header.Set("Content-Type", "application/ocsp-request")
ocspReq.Header.Set("Accept", "application/ocsp-response")
ocspResp, err := this.httpClient.Do(ocspReq)
if err != nil {
return nil, 0, errors.New("request ocsp failed: '" + ocspServerURL + "': " + err.Error())
}
defer func() {
_ = ocspResp.Body.Close()
}()
respData, err := ioutil.ReadAll(ocspResp.Body)
if err != nil {
return nil, 0, errors.New("read ocsp failed: '" + ocspServerURL + "': " + err.Error())
}
ocspResult, err := ocsp.ParseResponse(respData, issuerCert)
if err != nil {
return nil, 0, errors.New("decode ocsp failed: " + err.Error())
}
// 只返回Good的ocsp
if ocspResult.Status == ocsp.Good {
return respData, ocspResult.NextUpdate.Unix(), nil
}
return nil, 0, nil
}

View File

@@ -0,0 +1,19 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package tasks_test
import (
"github.com/TeaOSLab/EdgeAPI/internal/tasks"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestSSLCertUpdateOCSPTask_Loop(t *testing.T) {
dbs.NotifyReady()
var task = tasks.NewSSLCertUpdateOCSPTask()
err := task.Loop(false)
if err != nil {
t.Fatal(err)
}
}