Compare commits

...

169 Commits

Author SHA1 Message Date
刘祥超
6b81ffd074 边缘节点版本调整为0.3.1 2021-09-23 20:59:21 +08:00
刘祥超
eff3c77551 分组配置信息中增加分组ID 2021-09-23 14:41:32 +08:00
刘祥超
2089bac52f 可以在分组中设置一些全局配置选项 2021-09-22 19:39:43 +08:00
刘祥超
67760a53ba 域名解析任务可以使用集群ID筛选 2021-09-21 10:56:53 +08:00
刘祥超
d4d7b1fff7 可以设置集群的DNS记录TTL 2021-09-20 20:01:21 +08:00
刘祥超
d121bc86a0 在集群中可以设置自动加入DNS的CNAME记录 2021-09-20 16:37:48 +08:00
刘祥超
7a1bd29f6f 反向代理源站实现使用域名分组 2021-09-20 11:54:45 +08:00
刘祥超
f1af151080 开源版本也显示域名排行 2021-09-19 16:10:34 +08:00
刘祥超
f39d106bef 实现连通性变化发送通知功能 2021-09-18 14:21:56 +08:00
刘祥超
d9c092cd31 修复创建默认集群时没有写入API令牌的Bug 2021-09-16 15:43:07 +08:00
刘祥超
7d1d138e42 优化域名排行查询速度 2021-09-16 09:23:36 +08:00
刘祥超
89bd70819f 增加查找区域监控对象相关结果API 2021-09-15 17:53:34 +08:00
刘祥超
15156b68e3 修复IP地址不能修改在线状态的Bug 2021-09-15 11:45:00 +08:00
刘祥超
92a3b8f375 健康检查只检查启用的IP地址 2021-09-15 10:44:25 +08:00
刘祥超
6a152b7775 调整部分命名 2021-09-14 19:38:44 +08:00
刘祥超
ea915582c1 IP阈值动作增加WebHook 2021-09-14 15:27:48 +08:00
刘祥超
51b938b9c7 IP阈值增加节点分组和集群相关统计项目 2021-09-14 11:36:22 +08:00
刘祥超
c2f559d48e 优化节点设置交互 2021-09-13 16:47:40 +08:00
刘祥超
a96c1434b6 健康检查支持IPv6地址的节点 2021-09-13 13:46:20 +08:00
刘祥超
d135442a52 IP地址支持手动上线和从备用IP中恢复 2021-09-13 13:45:58 +08:00
刘祥超
0b8501a724 实现节点自动切换到备用IP 2021-09-13 10:51:05 +08:00
刘祥超
7fcc2b7dba 实现基础的IP地址阈值 2021-09-12 20:21:42 +08:00
刘祥超
1ea7fe0213 实现基本的监控终端管理 2021-09-08 19:34:31 +08:00
刘祥超
838d33648f 通过DNS方式申请ACME证书时支持二级域名 2021-09-08 18:23:37 +08:00
刘祥超
6cf4a8ea3e 对域名统计进行分表处理 2021-09-08 17:32:08 +08:00
刘祥超
15c40d6c96 版本号修改0.3.1 2021-09-08 17:31:18 +08:00
刘祥超
6dfecd69b4 提供区域监控上报结果接口 2021-09-06 08:12:48 +08:00
刘祥超
dbc97bc8de 实现基本的区域监控终端管理功能 2021-09-05 11:10:18 +08:00
刘祥超
8308e2e83d 修复边缘节点无法下载文件的Bug 2021-09-03 15:15:27 +08:00
刘祥超
8227138168 修复DNS节点升级文件无法下载的Bug 2021-08-31 22:36:36 +08:00
刘祥超
2227a14ba4 增加独立的IP地址管理功能 2021-08-31 17:24:52 +08:00
刘祥超
6137b44408 企业认证信息中增加节点数限制 2021-08-30 18:57:11 +08:00
刘祥超
f654c65626 查询指标数据时增加索引 2021-08-30 15:23:51 +08:00
刘祥超
674574a6af Update go.mod 2021-08-30 10:56:44 +08:00
刘祥超
f02ab1aae2 优化数据库节点管理 2021-08-30 10:56:31 +08:00
刘祥超
55527bba09 健康检查失败10分钟内不重复提醒 2021-08-30 09:47:17 +08:00
刘祥超
821b607ef2 当修改节点在线状态时重置上线检查次数 2021-08-29 16:57:50 +08:00
刘祥超
afeee89a88 节点返回数据增加isUp字段 2021-08-29 16:57:20 +08:00
刘祥超
a983615464 修复节点转移集群后没有删除老的DNS记录的问题 2021-08-29 16:41:42 +08:00
刘祥超
bd596816d5 将健康检查连续下线次数从1升级为3/修复健康检查可能导致DNS不断同步的问题 2021-08-29 16:01:31 +08:00
刘祥超
ca233c3573 修复一个因为SQL_CACHE而导致子查询产生的错误 2021-08-29 14:07:12 +08:00
刘祥超
fd1f990a0e 访问日志表增加requestBody, responseBody(预留) 2021-08-29 10:37:42 +08:00
刘祥超
1796bb8f96 更新TeaGo,提升SQL解析效率、自动开启SQL查询缓存 2021-08-29 10:21:42 +08:00
刘祥超
3f5e4babc7 改进编译脚本 2021-08-26 16:29:19 +08:00
刘祥超
f508c16f92 删除plus文件 2021-08-26 14:40:07 +08:00
刘祥超
ac0bbd0b99 DNS服务商支持搜索 2021-08-25 18:47:01 +08:00
刘祥超
9017176efb 集群支持使用域名搜索 2021-08-25 18:39:17 +08:00
刘祥超
8b40634e74 节点如果没有设置DNS线路就使用默认线路 2021-08-25 17:16:24 +08:00
刘祥超
cf476f79d6 Admin看板增加默认集群ID 2021-08-25 11:41:23 +08:00
刘祥超
fc38a6ab7e 创建集群时自动创建缓存策略和WAF策略 2021-08-25 11:18:37 +08:00
刘祥超
e4f0dafc1a 增加忽略相似消息周期设置 2021-08-24 20:45:12 +08:00
刘祥超
b7fda0b9cc 消息接收人可以设置接收消息时间段 2021-08-24 17:46:11 +08:00
刘祥超
f4cc5aa087 通知媒介可以设置发送频率 2021-08-24 15:46:53 +08:00
刘祥超
f58724065d 通知媒介增加任务队列查看功能 2021-08-24 14:22:44 +08:00
刘祥超
1e6b42c00c 优化WAF日志查询速度 2021-08-22 16:20:40 +08:00
刘祥超
8d759a104b 优化WAF日志访问速度 2021-08-22 16:00:32 +08:00
刘祥超
72d7ceb94e 提升节点组合配置效率 2021-08-22 11:35:33 +08:00
刘祥超
53f7a0b77e Dashboard可以提示API节点升级 2021-08-21 19:43:46 +08:00
刘祥超
56000a8b8a 优化服务配置更新机制 2021-08-21 17:24:29 +08:00
刘祥超
ab7b2fee3a 自建DNS支持递归查询 2021-08-21 16:46:41 +08:00
刘祥超
d768d46854 DNS访问日志显示匹配的线路 2021-08-20 11:27:16 +08:00
刘祥超
b86c9aad6f 节点排行增加条数限制 2021-08-20 10:10:55 +08:00
刘祥超
df667c6ee6 添加DNS账号时自动读取DNS服务商下域名 2021-08-19 14:26:34 +08:00
刘祥超
70331805d7 节点IP地址可以设置阈值 2021-08-18 16:19:16 +08:00
刘祥超
71dbf86572 节点IP增加是否启用、是否在线状态 2021-08-18 09:24:18 +08:00
刘祥超
0df358d70d 调整版本为0.3.0 2021-08-17 09:44:22 +08:00
刘祥超
93a17ced7c 上传SQL 2021-08-15 20:20:06 +08:00
刘祥超
580d09ef99 IP库查询结果显示城市 2021-08-15 20:08:04 +08:00
刘祥超
fed725d45c 增加通过IP来搜索IP名单的API 2021-08-15 15:42:32 +08:00
刘祥超
350b514fc7 改进细节 2021-08-15 10:39:41 +08:00
刘祥超
71d2671c04 增加SSH认证建议接口 2021-08-14 21:33:17 +08:00
刘祥超
5a22146309 增加SSH登录建议端口接口 2021-08-14 18:07:20 +08:00
刘祥超
ac2fb4c84b 可以远程停止和启动DNS节点 2021-08-12 11:47:39 +08:00
刘祥超
c25e3f18e0 修复边缘节点和DNS节点安装文件冲突的问题 2021-08-11 21:14:01 +08:00
刘祥超
d3e4f28c69 实现DNS节点远程安装 2021-08-11 21:00:29 +08:00
刘祥超
363892efb2 改进脚本 2021-08-11 17:16:54 +08:00
刘祥超
cf36559eea 访问日志显示节点信息 2021-08-10 11:15:15 +08:00
刘祥超
fa3e0ca6ab 自建DNS增加解析测试 2021-08-09 18:41:30 +08:00
刘祥超
7229b0db34 NS日志增加remoteAddr字段 2021-08-09 15:19:38 +08:00
刘祥超
23871804b1 EdgeDNS支持内置线路 2021-08-09 13:57:58 +08:00
刘祥超
5e00bfa4c1 优化节点到API节点连接管理 2021-08-08 16:17:25 +08:00
刘祥超
473a2db335 数据有更改时发送通知 2021-08-08 15:47:48 +08:00
刘祥超
c893de8af7 DNS节点增加在线状态通知 2021-08-08 10:29:48 +08:00
刘祥超
a8cf04d178 访问日志搜索增加域名和IP搜索 2021-08-07 22:04:22 +08:00
刘祥超
e94a7f9a77 运行日志只显示已经设置集群的节点 2021-08-07 16:52:28 +08:00
刘祥超
ccf435ee8e 优化代码 2021-08-07 16:11:35 +08:00
刘祥超
7dc5c5f349 修复utils.DumpResponse()可能会忽略err的问题 2021-08-07 11:04:03 +08:00
刘祥超
566c04f080 边缘节点没有集群的时候视为删除 2021-08-07 10:12:17 +08:00
刘祥超
5a13c7663c 修复HTTPFirewallPolicyService.CheckHTTPFirewallPolicyIPStatus()可能panic的Bug 2021-08-06 14:48:05 +08:00
刘祥超
1df6d579d7 修改一处测试 2021-08-06 14:47:42 +08:00
刘祥超
20110495ab 修复unique key无法升级的问题 2021-08-06 14:22:17 +08:00
刘祥超
ce8d656d65 调整版本为0.2.9 2021-08-06 14:21:54 +08:00
刘祥超
186fe3c365 修复无法升级国家/地区数据的Bug 2021-08-05 20:36:16 +08:00
刘祥超
a9ce2f45df 修复导致无法安装的严重Bug 2021-08-05 19:53:54 +08:00
刘祥超
5105af9918 自建DNS增加全局配置 2021-08-05 16:08:01 +08:00
刘祥超
378c485219 统计节点分组中节点数量时判断节点集群是否存在 2021-08-05 11:28:58 +08:00
刘祥超
2465993e2c 域名解析支持华为云解析DNS 2021-08-04 22:14:54 +08:00
刘祥超
818c1c25a7 调整版本 2021-08-04 15:36:06 +08:00
刘祥超
77c67ccd3f 调整版本号 2021-08-03 14:00:44 +08:00
刘祥超
d855fdedde 修复一个DNS节点和边缘节点地址可能会冲突的问题 2021-08-03 13:35:29 +08:00
刘祥超
95c734c87a 调整版本号 2021-08-03 10:40:32 +08:00
刘祥超
89ff1927b7 使用upgrade命令升级数据库时增加日志显示 2021-08-02 15:23:02 +08:00
刘祥超
1a3aaf2846 节点运行日志增加记录源站ID 2021-08-01 21:54:44 +08:00
刘祥超
7d25019abb 更新SQL 2021-08-01 14:59:52 +08:00
刘祥超
4cfc7d5387 修改服务组合配置为动态 2021-08-01 14:56:08 +08:00
刘祥超
9245cf9cdb 各项统计支持单节点多集群 2021-08-01 11:13:46 +08:00
刘祥超
9e1e57dfd8 初步实现多集群共享节点 2021-07-31 22:23:11 +08:00
刘祥超
87f032bebd 使用ES存储访问日志时自动生成requestId 2021-07-29 17:34:44 +08:00
刘祥超
bda407fc3f 实现基本的访问日志策略 2021-07-29 16:50:59 +08:00
刘祥超
3b2f6060b8 上传统计数据时自动清理旧数据 2021-07-28 17:04:31 +08:00
刘祥超
a15ad7dd04 增加内置统计指标:请求来源统计 2021-07-26 16:50:34 +08:00
刘祥超
a415ef6070 指标图表可以设置忽略空值和其他对象值 2021-07-26 16:44:29 +08:00
刘祥超
d94a822f52 调整若干文字 2021-07-26 11:23:21 +08:00
刘祥超
9e3fb9cf66 版本调整为0.2.6 2021-07-26 11:23:12 +08:00
刘祥超
84a9916d3e 使用Sock管理进程启停 2021-07-25 17:46:47 +08:00
刘祥超
7d1ae0d242 更新SQL和编译脚本 2021-07-25 16:28:20 +08:00
刘祥超
3fbad22218 区分社区版和商业版 2021-07-25 15:46:12 +08:00
刘祥超
5fffc8a833 DNS支持TSIG 2021-07-25 15:08:17 +08:00
刘祥超
2c72d28c48 DNS服务支持密钥管理 2021-07-25 09:43:57 +08:00
刘祥超
45ea803e7a 获取DNS节点配置信息时增加节点UniqueId信息 2021-07-24 10:10:02 +08:00
刘祥超
edd4b59207 规则的内容长度从1024调整为4096 2021-07-23 10:58:07 +08:00
刘祥超
38ad11fda0 v0.2.5.1 2021-07-23 10:57:25 +08:00
刘祥超
961d9b4dbc DNS节点可以自动升级 2021-07-22 18:42:57 +08:00
刘祥超
6436e83d9e 更新SQL 2021-07-22 08:16:06 +08:00
刘祥超
b2d3d483e8 自动清理日志的时候也清理DNS访问日志 2021-07-21 22:12:52 +08:00
刘祥超
89ec81f6b1 修改无法在访问日志中搜索IP的Bug 2021-07-21 22:12:35 +08:00
刘祥超
5124169e28 编译脚本增加arm64 2021-07-21 22:11:59 +08:00
刘祥超
fe4eb3928e 设置 max_prepared_stmt_count 失败时提示更详细 2021-07-21 09:01:37 +08:00
刘祥超
514d968b20 修复访问日志无法查询IP的Bug 2021-07-21 08:08:31 +08:00
刘祥超
3cce13e671 域名排行增加条数限制 2021-07-21 08:08:16 +08:00
刘祥超
6d359b09f2 更新SQL 2021-07-20 22:24:30 +08:00
刘祥超
316b3485ca API节点列表API允许认证节点调用 2021-07-20 19:03:35 +08:00
刘祥超
31c20122b4 域名服务增加停用 /启用 2021-07-20 19:03:10 +08:00
刘祥超
7343d4bfac 增加API令牌相关API 2021-07-20 17:15:44 +08:00
刘祥超
330d5a3d21 在各个地方支持IPv6 2021-07-20 10:55:34 +08:00
刘祥超
709845064f 自动清理过期指标统计数据 2021-07-19 21:01:26 +08:00
刘祥超
be5a89e513 更新SQL 2021-07-19 20:38:59 +08:00
刘祥超
80328b9b5f 增加内置的统计指标 2021-07-19 20:38:30 +08:00
刘祥超
c6a09e131b Dashboard增加指标图表 2021-07-19 17:58:16 +08:00
刘祥超
f52ea4fa4f 优化通过IP查找日志 2021-07-18 17:09:06 +08:00
刘祥超
6cbda588f7 实现WAF通知和记录IP功能 2021-07-18 15:52:34 +08:00
刘祥超
f9e7c3a2e0 WAF支持更多动作 2021-07-14 22:46:23 +08:00
刘祥超
c370dada11 访问日志增加RemoteAddr字段以加速查询 2021-07-14 22:43:31 +08:00
刘祥超
8d71afc176 搜索节点时加入所在集群状态 2021-07-14 13:59:16 +08:00
刘祥超
a39a114316 IP测试时同时也检查绑定的IP名单 2021-07-13 15:49:16 +08:00
刘祥超
3f5c3568db 更新SQL 2021-07-13 15:10:19 +08:00
刘祥超
f50ff00f0b 路径规则改成路由规则 2021-07-13 14:30:30 +08:00
刘祥超
7caa5bee75 优化看板 2021-07-13 11:04:45 +08:00
刘祥超
bf5368929b 更新SQL 2021-07-12 17:37:44 +08:00
刘祥超
ff6d8d9ba3 实现数据看板--WAF 2021-07-12 16:57:02 +08:00
刘祥超
877d42a271 管理界面可以切换风格 2021-07-12 10:21:28 +08:00
刘祥超
21c51cbdc8 实现数据看板-DNS 2021-07-11 21:44:08 +08:00
刘祥超
45e4eb72ac 实现数据看板--用户 2021-07-11 18:05:57 +08:00
刘祥超
11c344eef4 看板增加缓存目录用量 2021-07-08 19:43:09 +08:00
刘祥超
17008d6b03 健康检查失败时,会自动尝试再次连接 2021-07-08 18:37:11 +08:00
刘祥超
5512efbb70 实现服务看板(企业版) 2021-07-07 19:55:37 +08:00
刘祥超
1e0a7612df 节点列表增加下行流量,节点列表可以按CPU、内存、下行流量排序 2021-07-07 15:18:48 +08:00
刘祥超
8589e0d373 实现节点看板(仅对企业版开放) 2021-07-06 20:06:34 +08:00
刘祥超
c5edaef356 优化API节点监听逻辑,提升兼容性
如果用户填写的GPRC监听端口中的地址无法监听,则尝试只监听端口
2021-07-06 15:19:39 +08:00
刘祥超
692eb04c93 DNS节点检查是否需要升级时,加入集群是否可用的条件 2021-07-06 10:27:14 +08:00
刘祥超
83d9a17c2b 集群看板增加指标相关图表 2021-07-05 16:37:53 +08:00
刘祥超
a4422ef7bd 更新SQL 2021-07-05 11:38:45 +08:00
刘祥超
e6e9fcc3c3 实现集群看板 2021-07-05 11:37:22 +08:00
刘祥超
7484243dff 实现基本的图表管理 2021-07-03 15:45:04 +08:00
刘祥超
907676d688 指标数据增加总和数据 2021-07-01 10:39:42 +08:00
刘祥超
54dbe1f3e4 实现基础的统计指标 2021-06-30 19:59:49 +08:00
刘祥超
98a2d61fd1 SSH认证--私钥认证方式增加用户名选项 2021-06-30 14:56:36 +08:00
刘祥超
e544e088be 统计创建浏览器、系统时限制名称长度 2021-06-29 19:38:14 +08:00
刘祥超
86c33256ca 修改版本为0.2.5 2021-06-28 10:32:06 +08:00
377 changed files with 19735 additions and 2700 deletions

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*_plus.go
*-plus.sh

View File

@@ -6,6 +6,8 @@ function build() {
DIST=$ROOT/"../dist/${NAME}"
OS=${1}
ARCH=${2}
TAG=${3}
NODE_ARCHITECTS=("amd64" "386" "arm64" "mips64" "mips64le")
if [ -z $OS ]; then
echo "usage: build.sh OS ARCH"
@@ -15,11 +17,14 @@ function build() {
echo "usage: build.sh OS ARCH"
exit
fi
if [ -z $TAG ]; then
TAG="community"
fi
VERSION=$(lookup-version $ROOT/../internal/const/const.go)
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
ZIP="${NAME}-${OS}-${ARCH}-${TAG}-v${VERSION}.zip"
# check edge-node
# build edge-node
NodeVersion=$(lookup-version $ROOT"/../../EdgeNode/internal/const/const.go")
echo "building edge-node v${NodeVersion} ..."
EDGE_NODE_BUILD_SCRIPT=$ROOT"/../../EdgeNode/build/build.sh"
@@ -29,18 +34,47 @@ function build() {
fi
cd $ROOT"/../../EdgeNode/build"
echo "=============================="
architects=("amd64" "386" "arm64" "mips64" "mips64le")
for arch in "${architects[@]}"; do
./build.sh linux $arch
for arch in "${NODE_ARCHITECTS[@]}"; do
if [ ! -f $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" ]; then
./build.sh linux $arch $TAG
else
echo "use built node linux/$arch/v${NodeVersion}"
fi
done
echo "=============================="
cd -
rm -f $ROOT/deploy/*.zip
for arch in "${architects[@]}"; do
cp $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-v${NodeVersion}.zip" $ROOT/deploy/
for arch in "${NODE_ARCHITECTS[@]}"; do
cp $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" $ROOT/deploy/edge-node-linux-${arch}-v${NodeVersion}.zip
done
# build edge-dns
if [ "$TAG" = "plus" ]; then
DNS_ROOT=$ROOT"/../../EdgeDNS"
if [ -d $DNS_ROOT ]; then
DNSNodeVersion=$(lookup-version $ROOT"/../../EdgeDNS/internal/const/const.go")
echo "building edge-dns ${DNSNodeVersion} ..."
EDGE_DNS_NODE_BUILD_SCRIPT=$ROOT"/../../EdgeDNS/build/build.sh"
if [ ! -f $EDGE_DNS_NODE_BUILD_SCRIPT ]; then
echo "unable to find edge-dns build script 'EdgeDNS/build/build.sh'"
exit
fi
cd $ROOT"/../../EdgeDNS/build"
echo "=============================="
architects=("amd64")
for arch in "${architects[@]}"; do
./build.sh linux $arch $TAG
done
echo "=============================="
cd -
for arch in "${architects[@]}"; do
cp $ROOT"/../../EdgeDNS/dist/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip" $ROOT/deploy/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip
done
fi
fi
# build sql
echo "building sql ..."
${ROOT}/sql.sh
@@ -62,16 +96,24 @@ function build() {
rm -f $DIST/resources/ipdata/ip2region/global_region.csv
rm -f $DIST/resources/ipdata/ip2region/ip.merge.txt
# building installer
echo "building installer ..."
architects=("amd64" "386")
# building edge installer
echo "building node installer ..."
architects=("amd64" "386" "arm64")
for arch in "${architects[@]}"; do
# TODO support arm, mips ...
env GOOS=linux GOARCH=${arch} go build --ldflags="-s -w" -o $ROOT/installers/edge-installer-helper-linux-${arch} $ROOT/../cmd/installer-helper/main.go
env GOOS=linux GOARCH=${arch} go build -tags $TAG --ldflags="-s -w" -o $ROOT/installers/edge-installer-helper-linux-${arch} $ROOT/../cmd/installer-helper/main.go
done
# building edge dns installer
echo "building dns node installer ..."
architects=("amd64" "386" "arm64")
for arch in "${architects[@]}"; do
# TODO support arm, mips ...
env GOOS=linux GOARCH=${arch} go build -tags $TAG --ldflags="-s -w" -o $ROOT/installers/edge-installer-dns-helper-linux-${arch} $ROOT/../cmd/installer-dns-helper/main.go
done
# building api node
env GOOS=$OS GOARCH=$ARCH go build --ldflags="-s -w" -o $DIST/bin/edge-api $ROOT/../cmd/edge-api/main.go
env GOOS=$OS GOARCH=$ARCH go build -tags $TAG --ldflags="-s -w" -o $DIST/bin/edge-api $ROOT/../cmd/edge-api/main.go
# delete hidden files
find $DIST -name ".DS_Store" -delete
@@ -102,4 +144,4 @@ function lookup-version() {
fi
}
build $1 $2
build $1 $2 $3

View File

@@ -44,12 +44,13 @@ func main() {
_, _ = os.Stdout.Write(resultJSON)
})
app.On("upgrade", func() {
fmt.Println("start ...")
executor, err := setup.NewSQLExecutorFromCmd()
if err != nil {
fmt.Println("ERROR: " + err.Error())
return
}
err = executor.Run()
err = executor.Run(true)
if err != nil {
fmt.Println("ERROR: " + err.Error())
return

View File

@@ -0,0 +1,73 @@
package main
import (
"flag"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/iwind/gosock/pkg/gosock"
"os"
"os/exec"
)
func main() {
cmd := ""
flag.StringVar(&cmd, "cmd", "", "command name: [unzip]")
// unzip
zipPath := ""
targetPath := ""
flag.StringVar(&zipPath, "zip", "", "zip path")
flag.StringVar(&targetPath, "target", "", "target dir")
// parse
flag.Parse()
if len(cmd) == 0 {
stderr("need '-cmd=COMMAND' argument")
} else if cmd == "test" {
// 检查是否正在运行
var sock = gosock.NewTmpSock("edge-dns")
if sock.IsListening() {
// 从systemd中停止
systemctl, _ := exec.LookPath("systemctl")
if len(systemctl) > 0 {
systemctlCmd := exec.Command(systemctl, "stop", "edge-dns")
_ = systemctlCmd.Run()
}
// 从进程中停止
if sock.IsListening() {
_, _ = sock.Send(&gosock.Command{
Code: "stop",
})
}
}
} else if cmd == "unzip" { // 解压
if len(zipPath) == 0 {
stderr("ERROR: need '-zip=PATH' argument")
return
}
if len(targetPath) == 0 {
stderr("ERROR: need '-target=TARGET' argument")
return
}
unzip := utils.NewUnzip(zipPath, targetPath)
err := unzip.Run()
if err != nil {
stderr("ERROR: " + err.Error())
return
}
stdout("ok")
} else {
stderr("ERROR: not recognized command '" + cmd + "'")
}
}
func stdout(s string) {
_, _ = os.Stdout.WriteString(s + "\n")
}
func stderr(s string) {
_, _ = os.Stderr.WriteString(s + "\n")
}

View File

@@ -3,8 +3,9 @@ package main
import (
"flag"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"net"
"github.com/iwind/gosock/pkg/gosock"
"os"
"os/exec"
)
func main() {
@@ -24,11 +25,21 @@ func main() {
stderr("need '-cmd=COMMAND' argument")
} else if cmd == "test" {
// 检查是否正在运行
path := os.TempDir() + "/edge-node.sock"
conn, err := net.Dial("unix", path)
if err == nil {
_ = conn.Close()
stderr("test node status: edge node is running now, can not install again")
var sock = gosock.NewTmpSock("edge-node")
if sock.IsListening() {
// 从systemd中停止
systemctl, _ := exec.LookPath("systemctl")
if len(systemctl) > 0 {
systemctlCmd := exec.Command(systemctl, "stop", "edge-node")
_ = systemctlCmd.Run()
}
// 从进程中停止
if sock.IsListening() {
_, _ = sock.Send(&gosock.Command{
Code: "stop",
})
}
}
} else if cmd == "unzip" { // 解压
if len(zipPath) == 0 {

6
go.mod
View File

@@ -4,6 +4,7 @@ go 1.15
replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
require (
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
@@ -14,7 +15,9 @@ require (
github.com/go-sql-driver/mysql v1.5.0
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/golang/protobuf v1.5.2
github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f
github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
github.com/json-iterator/go v1.1.11 // indirect
github.com/lionsoul2014/ip2region v2.2.0-release+incompatible
github.com/mozillazg/go-pinyin v0.18.0
github.com/pkg/sftp v1.12.0
@@ -26,5 +29,4 @@ require (
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced // indirect
google.golang.org/grpc v1.38.0
google.golang.org/protobuf v1.26.0
gopkg.in/yaml.v2 v2.4.0 // indirect
)

38
go.sum
View File

@@ -134,7 +134,6 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
@@ -146,8 +145,8 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -181,17 +180,29 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4=
github.com/iwind/TeaGo v0.0.0-20200923021120-f5d76441fe9e/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f h1:r2O8PONj/KiuZjJHVHn7KlCePUIjNtgAmvLfgRafQ8o=
github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210806054428-5534da0db9d1 h1:AZKkwTNEZYrpyv62zIkxpLJsWhfOS7OEFovAcwd0aco=
github.com/iwind/TeaGo v0.0.0-20210806054428-5534da0db9d1/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210809112119-a57ed0e84e34 h1:ZCNQXLiGF5Z1cV3Pi03zCWzwwjPfsI5XhcrNhTvCFIU=
github.com/iwind/TeaGo v0.0.0-20210809112119-a57ed0e84e34/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210824034952-1a56ad7d0b5e h1:GDCU57lQD6W9u5KT2834MmK022FSeAbskb7H0p2eaJY=
github.com/iwind/TeaGo v0.0.0-20210824034952-1a56ad7d0b5e/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210829020150-9c36d31301a5 h1:ybjIXGT3E/ZbfkRhIb903WMfLyt2Uv5p4niAqi8jwvM=
github.com/iwind/TeaGo v0.0.0-20210829020150-9c36d31301a5/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13 h1:HuEJ5xJfujW1Q6rNDhOu5LQXEBB2qLPah3jYslT8Gz4=
github.com/iwind/TeaGo v0.0.0-20210831140440-a2a442471b13/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -288,8 +299,6 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS7IBEjKVSrjg=
github.com/shirou/gopsutil v2.20.9+incompatible h1:msXs2frUV+O/JLva9EDLpuJ84PrFsdCTCQex8PUdtkQ=
github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc=
github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -399,8 +408,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -414,8 +423,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -451,9 +460,9 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -464,7 +473,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -511,8 +519,8 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -549,7 +557,6 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced h1:c5geK1iMU3cDKtFrCVQIcjR3W+JOZMuhIyICMCTbtus=
google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
@@ -564,7 +571,6 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
@@ -576,7 +582,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
@@ -608,9 +613,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -0,0 +1,66 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package accesslogs
import (
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"strconv"
"time"
)
type BaseStorage struct {
isOk bool
version int
}
func (this *BaseStorage) SetVersion(version int) {
this.version = version
}
func (this *BaseStorage) Version() int {
return this.version
}
func (this *BaseStorage) IsOk() bool {
return this.isOk
}
func (this *BaseStorage) SetOk(isOk bool) {
this.isOk = isOk
}
// Marshal 对日志进行编码
func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) {
return json.Marshal(accessLog)
}
// FormatVariables 格式化字符串中的变量
func (this *BaseStorage) FormatVariables(s string) string {
now := time.Now()
return configutils.ParseVariables(s, func(varName string) (value string) {
switch varName {
case "year":
return strconv.Itoa(now.Year())
case "month":
return fmt.Sprintf("%02d", now.Month())
case "week":
_, week := now.ISOWeek()
return fmt.Sprintf("%02d", week)
case "day":
return fmt.Sprintf("%02d", now.Day())
case "hour":
return fmt.Sprintf("%02d", now.Hour())
case "minute":
return fmt.Sprintf("%02d", now.Minute())
case "second":
return fmt.Sprintf("%02d", now.Second())
case "date":
return fmt.Sprintf("%d%02d%02d", now.Year(), now.Month(), now.Day())
}
return varName
})
}

View File

@@ -0,0 +1,95 @@
package accesslogs
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/logs"
"os/exec"
"sync"
)
// CommandStorage 通过命令行存储
type CommandStorage struct {
BaseStorage
config *serverconfigs.AccessLogCommandStorageConfig
writeLocker sync.Mutex
}
func NewCommandStorage(config *serverconfigs.AccessLogCommandStorageConfig) *CommandStorage {
return &CommandStorage{config: config}
}
func (this *CommandStorage) Config() interface{} {
return this.config
}
// Start 启动
func (this *CommandStorage) Start() error {
if len(this.config.Command) == 0 {
return errors.New("'command' should not be empty")
}
return nil
}
// 写入日志
func (this *CommandStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
if len(accessLogs) == 0 {
return nil
}
this.writeLocker.Lock()
defer this.writeLocker.Unlock()
cmd := exec.Command(this.config.Command, this.config.Args...)
if len(this.config.Dir) > 0 {
cmd.Dir = this.config.Dir
}
stdout := bytes.NewBuffer([]byte{})
cmd.Stdout = stdout
w, err := cmd.StdinPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
for _, accessLog := range accessLogs {
data, err := this.Marshal(accessLog)
if err != nil {
logs.Error(err)
continue
}
_, err = w.Write(data)
if err != nil {
logs.Error(err)
}
_, err = w.Write([]byte("\n"))
if err != nil {
logs.Error(err)
}
}
_ = w.Close()
err = cmd.Wait()
if err != nil {
logs.Error(err)
if stdout.Len() > 0 {
logs.Error(errors.New(string(stdout.Bytes())))
}
}
return nil
}
// Close 关闭
func (this *CommandStorage) Close() error {
return nil
}

View File

@@ -0,0 +1,63 @@
package accesslogs
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"os"
"os/exec"
"testing"
"time"
)
func TestCommandStorage_Write(t *testing.T) {
php, err := exec.LookPath("php")
if err != nil { // not found php, so we can not test
t.Log("php:", err)
return
}
cwd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
before := time.Now()
storage := NewCommandStorage(&serverconfigs.AccessLogCommandStorageConfig{
Command: php,
Args: []string{cwd + "/tests/command_storage.php"},
})
err = storage.Start()
if err != nil {
t.Fatal(err)
}
err = storage.Write([]*pb.HTTPAccessLog{
{
RequestMethod: "GET",
RequestPath: "/hello",
},
{
RequestMethod: "GET",
RequestPath: "/world",
},
{
RequestMethod: "GET",
RequestPath: "/lu",
},
{
RequestMethod: "GET",
RequestPath: "/ping",
},
})
if err != nil {
t.Fatal(err)
}
err = storage.Close()
if err != nil {
t.Fatal(err)
}
t.Log(time.Since(before).Seconds(), "seconds")
}

View File

@@ -0,0 +1,131 @@
package accesslogs
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"io/ioutil"
"net/http"
"regexp"
"strconv"
"strings"
"sync/atomic"
"time"
)
// ESStorage ElasticSearch存储策略
type ESStorage struct {
BaseStorage
config *serverconfigs.AccessLogESStorageConfig
}
func NewESStorage(config *serverconfigs.AccessLogESStorageConfig) *ESStorage {
return &ESStorage{config: config}
}
func (this *ESStorage) Config() interface{} {
return this.config
}
// Start 开启
func (this *ESStorage) Start() error {
if len(this.config.Endpoint) == 0 {
return errors.New("'endpoint' should not be nil")
}
if !regexp.MustCompile(`(?i)^(http|https)://`).MatchString(this.config.Endpoint) {
this.config.Endpoint = "http://" + this.config.Endpoint
}
if len(this.config.Index) == 0 {
return errors.New("'index' should not be nil")
}
if len(this.config.MappingType) == 0 {
return errors.New("'mappingType' should not be nil")
}
return nil
}
// 写入日志
func (this *ESStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
if len(accessLogs) == 0 {
return nil
}
var requestId int64 = 1_0000_0000_0000_0000
bulk := &strings.Builder{}
indexName := this.FormatVariables(this.config.Index)
typeName := this.FormatVariables(this.config.MappingType)
for _, accessLog := range accessLogs {
if len(accessLog.RequestId) == 0 {
accessLog.RequestId = strconv.FormatInt(time.Now().UnixNano(), 10) + strconv.FormatInt(atomic.AddInt64(&requestId, 1), 10) + fmt.Sprintf("%08d", 1)
}
opData, err := json.Marshal(map[string]interface{}{
"index": map[string]interface{}{
"_index": indexName,
"_type": typeName,
"_id": accessLog.RequestId,
},
})
if err != nil {
remotelogs.Error("ACCESS_LOG_ES_STORAGE", "write failed: "+err.Error())
continue
}
data, err := this.Marshal(accessLog)
if err != nil {
remotelogs.Error("ACCESS_LOG_ES_STORAGE", "marshal data failed: "+err.Error())
continue
}
bulk.Write(opData)
bulk.WriteString("\n")
bulk.Write(data)
bulk.WriteString("\n")
}
if bulk.Len() == 0 {
return nil
}
req, err := http.NewRequest(http.MethodPost, this.config.Endpoint+"/_bulk", strings.NewReader(bulk.String()))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", strings.ReplaceAll(teaconst.ProductName, " ", "-")+"/"+teaconst.Version)
if len(this.config.Username) > 0 || len(this.config.Password) > 0 {
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(this.config.Username+":"+this.config.Password)))
}
client := utils.SharedHttpClient(10 * time.Second)
defer func() {
_ = req.Body.Close()
}()
resp, err := client.Do(req)
if err != nil {
return err
}
defer func() {
_ = resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
bodyData, _ := ioutil.ReadAll(resp.Body)
return errors.New("ElasticSearch response status code: " + fmt.Sprintf("%d", resp.StatusCode) + " content: " + string(bodyData))
}
return nil
}
// Close 关闭
func (this *ESStorage) Close() error {
return nil
}

View File

@@ -0,0 +1,53 @@
package accesslogs
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"testing"
"time"
)
func TestESStorage_Write(t *testing.T) {
storage := NewESStorage(&serverconfigs.AccessLogESStorageConfig{
Endpoint: "http://127.0.0.1:9200",
Index: "logs",
MappingType: "accessLogs",
Username: "hello",
Password: "world",
})
err := storage.Start()
if err != nil {
t.Fatal(err)
}
{
err = storage.Write([]*pb.HTTPAccessLog{
{
RequestMethod: "POST",
RequestPath: "/1",
TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"),
TimeISO8601: "2018-07-23T22:23:35+08:00",
Header: map[string]*pb.Strings{
"Content-Type": {Values: []string{"text/html"}},
},
},
{
RequestMethod: "GET",
RequestPath: "/2",
TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"),
TimeISO8601: "2018-07-23T22:23:35+08:00",
Header: map[string]*pb.Strings{
"Content-Type": {Values: []string{"text/css"}},
},
},
})
if err != nil {
t.Fatal(err)
}
}
err = storage.Close()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,127 @@
package accesslogs
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/logs"
"os"
"path/filepath"
"sync"
)
// FileStorage 文件存储策略
type FileStorage struct {
BaseStorage
config *serverconfigs.AccessLogFileStorageConfig
writeLocker sync.Mutex
files map[string]*os.File // path => *File
filesLocker sync.Mutex
}
func NewFileStorage(config *serverconfigs.AccessLogFileStorageConfig) *FileStorage {
return &FileStorage{
config: config,
}
}
func (this *FileStorage) Config() interface{} {
return this.config
}
// Start 开启
func (this *FileStorage) Start() error {
if len(this.config.Path) == 0 {
return errors.New("'path' should not be empty")
}
this.files = map[string]*os.File{}
return nil
}
// Write 写入日志
func (this *FileStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
if len(accessLogs) == 0 {
return nil
}
fp := this.fp()
if fp == nil {
return errors.New("file pointer should not be nil")
}
this.writeLocker.Lock()
defer this.writeLocker.Unlock()
for _, accessLog := range accessLogs {
data, err := this.Marshal(accessLog)
if err != nil {
logs.Error(err)
continue
}
_, err = fp.Write(data)
if err != nil {
_ = this.Close()
break
}
_, _ = fp.WriteString("\n")
}
return nil
}
// Close 关闭
func (this *FileStorage) Close() error {
this.filesLocker.Lock()
defer this.filesLocker.Unlock()
var resultErr error
for _, f := range this.files {
err := f.Close()
if err != nil {
resultErr = err
}
}
return resultErr
}
func (this *FileStorage) fp() *os.File {
path := this.FormatVariables(this.config.Path)
this.filesLocker.Lock()
defer this.filesLocker.Unlock()
fp, ok := this.files[path]
if ok {
return fp
}
// 关闭其他的文件
for _, f := range this.files {
_ = f.Close()
}
// 是否创建文件目录
if this.config.AutoCreate {
dir := filepath.Dir(path)
_, err := os.Stat(dir)
if os.IsNotExist(err) {
err = os.MkdirAll(dir, 0777)
if err != nil {
logs.Error(err)
return nil
}
}
}
// 打开新文件
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
logs.Error(err)
return nil
}
this.files[path] = fp
return fp
}

View File

@@ -0,0 +1,70 @@
package accesslogs
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/Tea"
"testing"
"time"
)
func TestFileStorage_Write(t *testing.T) {
storage := NewFileStorage(&serverconfigs.AccessLogFileStorageConfig{
Path: Tea.Root + "/logs/access-${date}.log",
})
err := storage.Start()
if err != nil {
t.Fatal(err)
}
{
err = storage.Write([]*pb.HTTPAccessLog{
{
RequestPath: "/hello",
},
{
RequestPath: "/world",
},
})
if err != nil {
t.Fatal(err)
}
}
{
err = storage.Write([]*pb.HTTPAccessLog{
{
RequestPath: "/1",
},
{
RequestPath: "/2",
},
})
if err != nil {
t.Fatal(err)
}
}
{
err = storage.Write([]*pb.HTTPAccessLog{
{
RequestMethod: "POST",
RequestPath: "/1",
TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"),
},
{
RequestMethod: "GET",
RequestPath: "/2",
TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"),
},
})
if err != nil {
t.Fatal(err)
}
}
err = storage.Close()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,30 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package accesslogs
import "github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
// StorageInterface 日志存储接口
type StorageInterface interface {
// Version 获取版本
Version() int
// SetVersion 设置版本
SetVersion(version int)
IsOk() bool
SetOk(ok bool)
// Config 获取配置
Config() interface{}
// Start 开启
Start() error
// Write 写入日志
Write(accessLogs []*pb.HTTPAccessLog) error
// Close 关闭
Close() error
}

View File

@@ -0,0 +1,199 @@
package accesslogs
import (
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
"sync"
"time"
)
var SharedStorageManager = NewStorageManager()
type StorageManager struct {
storageMap map[int64]StorageInterface // policyId => Storage
locker sync.Mutex
}
func NewStorageManager() *StorageManager {
return &StorageManager{
storageMap: map[int64]StorageInterface{},
}
}
func (this *StorageManager) Start() {
var ticker = time.NewTicker(1 * time.Minute)
if Tea.IsTesting() {
ticker = time.NewTicker(5 * time.Second)
}
// 启动时执行一次
var err = this.Loop()
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "update error: "+err.Error())
}
// 循环执行
for range ticker.C {
err := this.Loop()
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "update error: "+err.Error())
}
}
}
// 写入日志
func (this *StorageManager) Write(policyId int64, accessLogs []*pb.HTTPAccessLog) error {
this.locker.Lock()
storage, ok := this.storageMap[policyId]
this.locker.Unlock()
if !ok {
return nil
}
if !storage.IsOk() {
return nil
}
return storage.Write(accessLogs)
}
// Loop 更新
func (this *StorageManager) Loop() error {
policies, err := models.SharedHTTPAccessLogPolicyDAO.FindAllEnabledAndOnPolicies(nil)
if err != nil {
return err
}
var policyIds = []int64{}
for _, policy := range policies {
if policy.IsOn == 1 {
policyIds = append(policyIds, int64(policy.Id))
}
}
this.locker.Lock()
defer this.locker.Unlock()
// 关闭不用的
for policyId, storage := range this.storageMap {
if !lists.ContainsInt64(policyIds, policyId) {
err := storage.Close()
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "close '"+types.String(policyId)+"' failed: "+err.Error())
}
delete(this.storageMap, policyId)
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "remove '"+types.String(policyId)+"'")
}
}
for _, policy := range policies {
var policyId = int64(policy.Id)
storage, ok := this.storageMap[policyId]
if ok {
// 检查配置是否有变更
if types.Int(policy.Version) != storage.Version() {
err = storage.Close()
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "close policy '"+types.String(policyId)+"' failed: "+err.Error())
// 继续往下执行
}
if len(policy.Options) > 0 {
err = json.Unmarshal([]byte(policy.Options), storage.Config())
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "unmarshal policy '"+types.String(policyId)+"' config failed: "+err.Error())
storage.SetOk(false)
continue
}
}
storage.SetVersion(types.Int(policy.Version))
err := storage.Start()
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error())
continue
}
storage.SetOk(true)
remotelogs.Println("ACCESS_LOG_STORAGE_MANAGER", "restart policy '"+types.String(policyId)+"'")
}
} else {
storage, err := this.createStorage(policy.Type, []byte(policy.Options))
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "create policy '"+types.String(policyId)+"' failed: "+err.Error())
continue
}
storage.SetVersion(types.Int(policy.Version))
this.storageMap[policyId] = storage
err = storage.Start()
if err != nil {
remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error())
continue
}
storage.SetOk(true)
remotelogs.Println("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"'")
}
}
return nil
}
func (this *StorageManager) createStorage(storageType string, optionsJSON []byte) (StorageInterface, error) {
switch storageType {
case serverconfigs.AccessLogStorageTypeFile:
var config = &serverconfigs.AccessLogFileStorageConfig{}
if len(optionsJSON) > 0 {
err := json.Unmarshal(optionsJSON, config)
if err != nil {
return nil, err
}
}
return NewFileStorage(config), nil
case serverconfigs.AccessLogStorageTypeES:
var config = &serverconfigs.AccessLogESStorageConfig{}
if len(optionsJSON) > 0 {
err := json.Unmarshal(optionsJSON, config)
if err != nil {
return nil, err
}
}
return NewESStorage(config), nil
case serverconfigs.AccessLogStorageTypeTCP:
var config = &serverconfigs.AccessLogTCPStorageConfig{}
if len(optionsJSON) > 0 {
err := json.Unmarshal(optionsJSON, config)
if err != nil {
return nil, err
}
}
return NewTCPStorage(config), nil
case serverconfigs.AccessLogStorageTypeSyslog:
var config = &serverconfigs.AccessLogSyslogStorageConfig{}
if len(optionsJSON) > 0 {
err := json.Unmarshal(optionsJSON, config)
if err != nil {
return nil, err
}
}
return NewSyslogStorage(config), nil
case serverconfigs.AccessLogStorageTypeCommand:
var config = &serverconfigs.AccessLogCommandStorageConfig{}
if len(optionsJSON) > 0 {
err := json.Unmarshal(optionsJSON, config)
if err != nil {
return nil, err
}
}
return NewCommandStorage(config), nil
}
return nil, errors.New("invalid policy type '" + storageType + "'")
}

View File

@@ -0,0 +1,17 @@
package accesslogs
import (
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestStorageManager_Loop(t *testing.T) {
dbs.NotifyReady()
var storage = NewStorageManager()
err := storage.Loop()
if err != nil {
t.Fatal(err)
}
t.Log(storage.storageMap)
}

View File

@@ -0,0 +1,137 @@
package accesslogs
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/logs"
"os/exec"
"runtime"
"strconv"
)
type SyslogStorageProtocol = string
const (
SyslogStorageProtocolTCP SyslogStorageProtocol = "tcp"
SyslogStorageProtocolUDP SyslogStorageProtocol = "udp"
SyslogStorageProtocolNone SyslogStorageProtocol = "none"
SyslogStorageProtocolSocket SyslogStorageProtocol = "socket"
)
type SyslogStoragePriority = int
// SyslogStorage syslog存储策略
type SyslogStorage struct {
BaseStorage
config *serverconfigs.AccessLogSyslogStorageConfig
exe string
}
func NewSyslogStorage(config *serverconfigs.AccessLogSyslogStorageConfig) *SyslogStorage {
return &SyslogStorage{config: config}
}
func (this *SyslogStorage) Config() interface{} {
return this.config
}
// Start 开启
func (this *SyslogStorage) Start() error {
if runtime.GOOS != "linux" {
return errors.New("'syslog' storage only works on linux")
}
exe, err := exec.LookPath("logger")
if err != nil {
return err
}
this.exe = exe
return nil
}
// 写入日志
func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
if len(accessLogs) == 0 {
return nil
}
args := []string{}
if len(this.config.Tag) > 0 {
args = append(args, "-t", this.config.Tag)
}
if this.config.Priority >= 0 {
args = append(args, "-p", strconv.Itoa(this.config.Priority))
}
switch this.config.Protocol {
case SyslogStorageProtocolTCP:
args = append(args, "-T")
if len(this.config.ServerAddr) > 0 {
args = append(args, "-n", this.config.ServerAddr)
}
if this.config.ServerPort > 0 {
args = append(args, "-P", strconv.Itoa(this.config.ServerPort))
}
case SyslogStorageProtocolUDP:
args = append(args, "-d")
if len(this.config.ServerAddr) > 0 {
args = append(args, "-n", this.config.ServerAddr)
}
if this.config.ServerPort > 0 {
args = append(args, "-P", strconv.Itoa(this.config.ServerPort))
}
case SyslogStorageProtocolSocket:
args = append(args, "-u")
args = append(args, this.config.Socket)
case SyslogStorageProtocolNone:
// do nothing
}
args = append(args, "-S", "10240")
cmd := exec.Command(this.exe, args...)
w, err := cmd.StdinPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
for _, accessLog := range accessLogs {
data, err := this.Marshal(accessLog)
if err != nil {
logs.Error(err)
continue
}
_, err = w.Write(data)
if err != nil {
logs.Error(err)
}
_, err = w.Write([]byte("\n"))
if err != nil {
logs.Error(err)
}
}
_ = w.Close()
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
// Close 关闭
func (this *SyslogStorage) Close() error {
return nil
}

View File

@@ -0,0 +1,111 @@
package accesslogs
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/logs"
"net"
"sync"
)
// TCPStorage TCP存储策略
type TCPStorage struct {
BaseStorage
config *serverconfigs.AccessLogTCPStorageConfig
writeLocker sync.Mutex
connLocker sync.Mutex
conn net.Conn
}
func NewTCPStorage(config *serverconfigs.AccessLogTCPStorageConfig) *TCPStorage {
return &TCPStorage{config: config}
}
func (this *TCPStorage) Config() interface{} {
return this.config
}
// Start 开启
func (this *TCPStorage) Start() error {
if len(this.config.Network) == 0 {
return errors.New("'network' should not be empty")
}
if len(this.config.Addr) == 0 {
return errors.New("'addr' should not be empty")
}
return nil
}
// 写入日志
func (this *TCPStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
if len(accessLogs) == 0 {
return nil
}
err := this.connect()
if err != nil {
return err
}
conn := this.conn
if conn == nil {
return errors.New("connection should not be nil")
}
this.writeLocker.Lock()
defer this.writeLocker.Unlock()
for _, accessLog := range accessLogs {
data, err := this.Marshal(accessLog)
if err != nil {
logs.Error(err)
continue
}
_, err = conn.Write(data)
if err != nil {
_ = this.Close()
break
}
_, err = conn.Write([]byte("\n"))
if err != nil {
_ = this.Close()
break
}
}
return nil
}
// Close 关闭
func (this *TCPStorage) Close() error {
this.connLocker.Lock()
defer this.connLocker.Unlock()
if this.conn != nil {
err := this.conn.Close()
this.conn = nil
return err
}
return nil
}
func (this *TCPStorage) connect() error {
this.connLocker.Lock()
defer this.connLocker.Unlock()
if this.conn != nil {
return nil
}
conn, err := net.Dial(this.config.Network, this.config.Addr)
if err != nil {
return err
}
this.conn = conn
return nil
}

View File

@@ -0,0 +1,72 @@
package accesslogs
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"net"
"testing"
"time"
)
func TestTCPStorage_Write(t *testing.T) {
go func() {
server, err := net.Listen("tcp", "127.0.0.1:9981")
if err != nil {
t.Error(err)
return
}
for {
conn, err := server.Accept()
if err != nil {
break
}
buf := make([]byte, 1024)
for {
n, err := conn.Read(buf)
if n > 0 {
t.Log(string(buf[:n]))
}
if err != nil {
break
}
}
break
}
_ = server.Close()
}()
storage := NewTCPStorage(&serverconfigs.AccessLogTCPStorageConfig{
Network: "tcp",
Addr: "127.0.0.1:9981",
})
err := storage.Start()
if err != nil {
t.Fatal(err)
}
{
err = storage.Write([]*pb.HTTPAccessLog{
{
RequestMethod: "POST",
RequestPath: "/1",
TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"),
},
{
RequestMethod: "GET",
RequestPath: "/2",
TimeLocal: time.Now().Format("2/Jan/2006:15:04:05 -0700"),
},
})
if err != nil {
t.Fatal(err)
}
}
time.Sleep(2 * time.Second)
err = storage.Close()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,24 @@
<?php
// test command storage
// open access log file
$fp = fopen("/tmp/goedge-command-storage.log", "a+");
// read access logs from stdin
$stdin = fopen("php://stdin", "r");
while(true) {
if (feof($stdin)) {
break;
}
$line = fgets($stdin);
// write to access log file
fwrite($fp, $line);
}
// close file pointers
fclose($fp);
fclose($stdin);
?>

View File

@@ -9,28 +9,32 @@ import (
)
type DNSProvider struct {
raw dnsclients.ProviderInterface
raw dnsclients.ProviderInterface
dnsDomain string
}
func NewDNSProvider(raw dnsclients.ProviderInterface) *DNSProvider {
return &DNSProvider{raw: raw}
func NewDNSProvider(raw dnsclients.ProviderInterface, dnsDomain string) *DNSProvider {
return &DNSProvider{
raw: raw,
dnsDomain: dnsDomain,
}
}
func (this *DNSProvider) Present(domain, token, keyAuth string) error {
fqdn, value := dns01.GetRecord(domain, keyAuth)
// 设置记录
index := strings.Index(fqdn, "."+domain)
index := strings.Index(fqdn, "."+this.dnsDomain)
if index < 0 {
return errors.New("invalid fqdn value")
}
recordName := fqdn[:index]
record, err := this.raw.QueryRecord(domain, recordName, dnstypes.RecordTypeTXT)
record, err := this.raw.QueryRecord(this.dnsDomain, recordName, dnstypes.RecordTypeTXT)
if err != nil {
return errors.New("query DNS record failed: " + err.Error())
}
if record == nil {
err = this.raw.AddRecord(domain, &dnstypes.Record{
err = this.raw.AddRecord(this.dnsDomain, &dnstypes.Record{
Id: "",
Name: recordName,
Type: dnstypes.RecordTypeTXT,
@@ -41,7 +45,7 @@ func (this *DNSProvider) Present(domain, token, keyAuth string) error {
return errors.New("create DNS record failed: " + err.Error())
}
} else {
err = this.raw.UpdateRecord(domain, record, &dnstypes.Record{
err = this.raw.UpdateRecord(this.dnsDomain, record, &dnstypes.Record{
Name: recordName,
Type: dnstypes.RecordTypeTXT,
Value: value,

View File

@@ -92,7 +92,7 @@ func (this *Request) runDNS() (certData []byte, keyData []byte, err error) {
}
}
err = client.Challenge.SetDNS01Provider(NewDNSProvider(this.task.DNSProvider))
err = client.Challenge.SetDNS01Provider(NewDNSProvider(this.task.DNSProvider, this.task.DNSDomain))
if err != nil {
return nil, nil, err
}

View File

@@ -42,7 +42,7 @@ func TestRequest_Run_DNS(t *testing.T) {
AuthType: AuthTypeDNS,
DNSProvider: dnsProvider,
DNSDomain: "yun4s.cn",
Domains: []string{"yun4s.cn"},
Domains: []string{"www.yun4s.cn"},
})
certData, keyData, err := req.Run()
if err != nil {

View File

@@ -2,8 +2,11 @@ package apps
import (
"fmt"
"github.com/iwind/TeaGo/Tea"
teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"github.com/iwind/gosock/pkg/gosock"
"os"
"os/exec"
"runtime"
@@ -11,7 +14,7 @@ import (
"time"
)
// App命令帮助
// AppCmd App命令帮助
type AppCmd struct {
product string
version string
@@ -20,10 +23,14 @@ type AppCmd struct {
appendStrings []string
directives []*Directive
sock *gosock.Sock
}
func NewAppCmd() *AppCmd {
return &AppCmd{}
return &AppCmd{
sock: gosock.NewTmpSock(teaconst.ProcessName),
}
}
type CommandHelpOption struct {
@@ -31,25 +38,25 @@ type CommandHelpOption struct {
Description string
}
// 产品
// Product 产品
func (this *AppCmd) Product(product string) *AppCmd {
this.product = product
return this
}
// 版本
// Version 版本
func (this *AppCmd) Version(version string) *AppCmd {
this.version = version
return this
}
// 使用方法
// Usage 使用方法
func (this *AppCmd) Usage(usage string) *AppCmd {
this.usage = usage
return this
}
// 选项
// Option 选项
func (this *AppCmd) Option(code string, description string) *AppCmd {
this.options = append(this.options, &CommandHelpOption{
Code: code,
@@ -58,13 +65,13 @@ func (this *AppCmd) Option(code string, description string) *AppCmd {
return this
}
// 附加内容
// Append 附加内容
func (this *AppCmd) Append(appendString string) *AppCmd {
this.appendStrings = append(this.appendStrings, appendString)
return this
}
// 打印
// Print 打印
func (this *AppCmd) Print() {
fmt.Println(this.product + " v" + this.version)
@@ -103,7 +110,7 @@ func (this *AppCmd) Print() {
}
}
// 添加指令
// On 添加指令
func (this *AppCmd) On(arg string, callback func()) {
this.directives = append(this.directives, &Directive{
Arg: arg,
@@ -111,7 +118,7 @@ func (this *AppCmd) On(arg string, callback func()) {
})
}
// 运行
// Run 运行
func (this *AppCmd) Run(main func()) {
// 获取参数
args := os.Args[1:]
@@ -150,9 +157,6 @@ func (this *AppCmd) Run(main func()) {
return
}
// 记录PID
_ = this.writePid()
// 日志
writer := new(LogWriter)
writer.Init()
@@ -164,7 +168,7 @@ func (this *AppCmd) Run(main func()) {
// 版本号
func (this *AppCmd) runVersion() {
fmt.Println(this.product+" v"+this.version, "(build: "+runtime.Version(), runtime.GOOS, runtime.GOARCH+")")
fmt.Println(this.product+" v"+this.version, "(build: "+runtime.Version(), runtime.GOOS, runtime.GOARCH, teaconst.Tag+")")
}
// 帮助
@@ -174,9 +178,9 @@ func (this *AppCmd) runHelp() {
// 启动
func (this *AppCmd) runStart() {
proc := this.checkPid()
if proc != nil {
fmt.Println(this.product+" already started, pid:", proc.Pid)
var pid = this.getPID()
if pid > 0 {
fmt.Println(this.product+" already started, pid:", pid)
return
}
@@ -192,18 +196,15 @@ func (this *AppCmd) runStart() {
// 停止
func (this *AppCmd) runStop() {
proc := this.checkPid()
if proc == nil {
var pid = this.getPID()
if pid == 0 {
fmt.Println(this.product + " not started yet")
return
}
// 停止进程
_ = proc.Kill()
_, _ = this.sock.Send(&gosock.Command{Code: "stop"})
// 在Windows上经常不能及时释放资源
_ = DeletePid(Tea.Root + "/bin/pid")
fmt.Println(this.product+" stopped ok, pid:", proc.Pid)
fmt.Println(this.product+" stopped ok, pid:", types.String(pid))
}
// 重启
@@ -215,20 +216,24 @@ func (this *AppCmd) runRestart() {
// 状态
func (this *AppCmd) runStatus() {
proc := this.checkPid()
if proc == nil {
var pid = this.getPID()
if pid == 0 {
fmt.Println(this.product + " not started yet")
} else {
fmt.Println(this.product + " is running, pid: " + fmt.Sprintf("%d", proc.Pid))
return
}
fmt.Println(this.product + " is running, pid: " + types.String(pid))
}
// 检查PID
func (this *AppCmd) checkPid() *os.Process {
return CheckPid(Tea.Root + "/bin/pid")
}
// 获取当前的PID
func (this *AppCmd) getPID() int {
if !this.sock.IsListening() {
return 0
}
// 写入PID
func (this *AppCmd) writePid() error {
return WritePid(Tea.Root + "/bin/pid")
reply, err := this.sock.Send(&gosock.Command{Code: "pid"})
if err != nil {
return 0
}
return maps.NewMap(reply.Params).GetInt("pid")
}

View File

@@ -1,17 +0,0 @@
// +build !windows
package apps
import (
"os"
"syscall"
)
// lock file
func LockFile(fp *os.File) error {
return syscall.Flock(int(fp.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
}
func UnlockFile(fp *os.File) error {
return syscall.Flock(int(fp.Fd()), syscall.LOCK_UN)
}

View File

@@ -1,17 +0,0 @@
// +build windows
package apps
import (
"errors"
"os"
)
// lock file
func LockFile(fp *os.File) error {
return errors.New("not implemented on windows")
}
func UnlockFile(fp *os.File) error {
return errors.New("not implemented on windows")
}

View File

@@ -1,113 +0,0 @@
package apps
import (
"fmt"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"runtime"
)
var pidFileList = []*os.File{}
// 检查Pid
func CheckPid(path string) *os.Process {
// windows上打开的文件是不能删除的
if runtime.GOOS == "windows" {
if os.Remove(path) == nil {
return nil
}
}
file, err := os.Open(path)
if err != nil {
return nil
}
defer func() {
_ = file.Close()
}()
// 是否能取得Lock
err = LockFile(file)
if err == nil {
_ = UnlockFile(file)
return nil
}
pidBytes, err := ioutil.ReadAll(file)
if err != nil {
return nil
}
pid := types.Int(string(pidBytes))
if pid <= 0 {
return nil
}
proc, _ := os.FindProcess(pid)
return proc
}
// 写入Pid
func WritePid(path string) error {
fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666)
if err != nil {
return err
}
if runtime.GOOS != "windows" {
err = LockFile(fp)
if err != nil {
return err
}
}
pidFileList = append(pidFileList, fp) // hold the file pointers
_, err = fp.WriteString(fmt.Sprintf("%d", os.Getpid()))
if err != nil {
return err
}
return nil
}
// 写入Ppid
func WritePpid(path string) error {
fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_RDONLY, 0666)
if err != nil {
return err
}
if runtime.GOOS != "windows" {
err = LockFile(fp)
if err != nil {
return err
}
}
pidFileList = append(pidFileList, fp) // hold the file pointers
_, err = fp.WriteString(fmt.Sprintf("%d", os.Getppid()))
if err != nil {
return err
}
return nil
}
// 删除Pid
func DeletePid(path string) error {
_, err := os.Stat(path)
if err != nil {
if !os.IsNotExist(err) {
return nil
}
return err
}
for _, fp := range pidFileList {
_ = UnlockFile(fp)
_ = fp.Close()
}
return os.Remove(path)
}

8
internal/const/build.go Normal file
View File

@@ -0,0 +1,8 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
// +build community
package teaconst
const BuildCommunity = true
const BuildPlus = false
const Tag = "community"

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.2.4"
Version = "0.3.1"
ProductName = "Edge API"
ProcessName = "edge-api"
@@ -18,9 +18,10 @@ const (
// 其他节点版本号,用来检测是否有需要升级的节点
NodeVersion = "0.2.4"
UserNodeVersion = "0.0.9"
NodeVersion = "0.3.1"
UserNodeVersion = "0.0.10"
AuthorityNodeVersion = "0.0.2"
MonitorNodeVersion = "0.0.2"
DNSNodeVersion = "0.0.1"
MonitorNodeVersion = "0.0.3"
DNSNodeVersion = "0.2.0"
ReportNodeVersion = "0.1.0"
)

View File

@@ -3,5 +3,6 @@
package teaconst
var (
IsPlus = false
IsPlus = false
MaxNodes int32 = 0
)

View File

@@ -33,7 +33,7 @@ func TestDB_Instance(t *testing.T) {
if err == driver.ErrBadConn {
return
}
t.Fatal(i, "exec:", err)
t.Error(i, "exec:", err)
}
time.Sleep(1 * time.Second)
}

View File

@@ -35,7 +35,7 @@ func init() {
})
}
// 启用条目
// EnableAdmin 启用条目
func (this *AdminDAO) EnableAdmin(tx *dbs.Tx, id int64) (rowsAffected int64, err error) {
return this.Query(tx).
Pk(id).
@@ -43,7 +43,7 @@ func (this *AdminDAO) EnableAdmin(tx *dbs.Tx, id int64) (rowsAffected int64, err
Update()
}
// 禁用条目
// DisableAdmin 禁用条目
func (this *AdminDAO) DisableAdmin(tx *dbs.Tx, id int64) (rowsAffected int64, err error) {
return this.Query(tx).
Pk(id).
@@ -51,7 +51,7 @@ func (this *AdminDAO) DisableAdmin(tx *dbs.Tx, id int64) (rowsAffected int64, er
Update()
}
// 查找启用中的条目
// FindEnabledAdmin 查找启用中的条目
func (this *AdminDAO) FindEnabledAdmin(tx *dbs.Tx, id int64) (*Admin, error) {
result, err := this.Query(tx).
Pk(id).
@@ -63,7 +63,7 @@ func (this *AdminDAO) FindEnabledAdmin(tx *dbs.Tx, id int64) (*Admin, error) {
return result.(*Admin), err
}
// 检查管理员是否存在
// ExistEnabledAdmin 检查管理员是否存在
func (this *AdminDAO) ExistEnabledAdmin(tx *dbs.Tx, adminId int64) (bool, error) {
return this.Query(tx).
Pk(adminId).
@@ -71,7 +71,7 @@ func (this *AdminDAO) ExistEnabledAdmin(tx *dbs.Tx, adminId int64) (bool, error)
Exist()
}
// 获取管理员名称
// FindAdminFullname 获取管理员名称
func (this *AdminDAO) FindAdminFullname(tx *dbs.Tx, adminId int64) (string, error) {
return this.Query(tx).
Pk(adminId).
@@ -79,7 +79,7 @@ func (this *AdminDAO) FindAdminFullname(tx *dbs.Tx, adminId int64) (string, erro
FindStringCol("")
}
// 检查用户名、密码
// CheckAdminPassword 检查用户名、密码
func (this *AdminDAO) CheckAdminPassword(tx *dbs.Tx, username string, encryptedPassword string) (int64, error) {
if len(username) == 0 || len(encryptedPassword) == 0 {
return 0, nil
@@ -94,7 +94,7 @@ func (this *AdminDAO) CheckAdminPassword(tx *dbs.Tx, username string, encryptedP
FindInt64Col(0)
}
// 根据用户名查询管理员ID
// FindAdminIdWithUsername 根据用户名查询管理员ID
func (this *AdminDAO) FindAdminIdWithUsername(tx *dbs.Tx, username string) (int64, error) {
one, err := this.Query(tx).
Attr("username", username).
@@ -110,7 +110,7 @@ func (this *AdminDAO) FindAdminIdWithUsername(tx *dbs.Tx, username string) (int6
return int64(one.(*Admin).Id), nil
}
// 更改管理员密码
// UpdateAdminPassword 更改管理员密码
func (this *AdminDAO) UpdateAdminPassword(tx *dbs.Tx, adminId int64, password string) error {
if adminId <= 0 {
return errors.New("invalid adminId")
@@ -122,7 +122,7 @@ func (this *AdminDAO) UpdateAdminPassword(tx *dbs.Tx, adminId int64, password st
return err
}
// 创建管理员
// CreateAdmin 创建管理员
func (this *AdminDAO) CreateAdmin(tx *dbs.Tx, username string, canLogin bool, password string, fullname string, isSuper bool, modulesJSON []byte) (int64, error) {
op := NewAdminOperator()
op.IsOn = true
@@ -144,7 +144,7 @@ func (this *AdminDAO) CreateAdmin(tx *dbs.Tx, username string, canLogin bool, pa
return types.Int64(op.Id), nil
}
// 修改管理员个人资料
// UpdateAdminInfo 修改管理员个人资料
func (this *AdminDAO) UpdateAdminInfo(tx *dbs.Tx, adminId int64, fullname string) error {
if adminId <= 0 {
return errors.New("invalid adminId")
@@ -156,7 +156,7 @@ func (this *AdminDAO) UpdateAdminInfo(tx *dbs.Tx, adminId int64, fullname string
return err
}
// 修改管理员详细信息
// UpdateAdmin 修改管理员详细信息
func (this *AdminDAO) UpdateAdmin(tx *dbs.Tx, adminId int64, username string, canLogin bool, password string, fullname string, isSuper bool, modulesJSON []byte, isOn bool) error {
if adminId <= 0 {
return errors.New("invalid adminId")
@@ -180,7 +180,7 @@ func (this *AdminDAO) UpdateAdmin(tx *dbs.Tx, adminId int64, username string, ca
return err
}
// 检查用户名是否存在
// CheckAdminUsername 检查用户名是否存在
func (this *AdminDAO) CheckAdminUsername(tx *dbs.Tx, adminId int64, username string) (bool, error) {
query := this.Query(tx).
State(AdminStateEnabled).
@@ -193,7 +193,7 @@ func (this *AdminDAO) CheckAdminUsername(tx *dbs.Tx, adminId int64, username str
return query.Exist()
}
// 修改管理员登录信息
// UpdateAdminLogin 修改管理员登录信息
func (this *AdminDAO) UpdateAdminLogin(tx *dbs.Tx, adminId int64, username string, password string) error {
if adminId <= 0 {
return errors.New("invalid adminId")
@@ -208,7 +208,7 @@ func (this *AdminDAO) UpdateAdminLogin(tx *dbs.Tx, adminId int64, username strin
return err
}
// 修改管理员可以管理的模块
// UpdateAdminModules 修改管理员可以管理的模块
func (this *AdminDAO) UpdateAdminModules(tx *dbs.Tx, adminId int64, allowModulesJSON []byte) error {
if adminId <= 0 {
return errors.New("invalid adminId")
@@ -223,25 +223,25 @@ func (this *AdminDAO) UpdateAdminModules(tx *dbs.Tx, adminId int64, allowModules
return nil
}
// 查询所有管理的权限
// FindAllAdminModules 查询所有管理的权限
func (this *AdminDAO) FindAllAdminModules(tx *dbs.Tx) (result []*Admin, err error) {
_, err = this.Query(tx).
State(AdminStateEnabled).
Attr("isOn", true).
Result("id", "modules", "isSuper", "fullname").
Result("id", "modules", "isSuper", "fullname", "theme").
Slice(&result).
FindAll()
return
}
// 计算所有管理员数量
// CountAllEnabledAdmins 计算所有管理员数量
func (this *AdminDAO) CountAllEnabledAdmins(tx *dbs.Tx) (int64, error) {
return this.Query(tx).
State(AdminStateEnabled).
Count()
}
// 列出单页的管理员
// ListEnabledAdmins 列出单页的管理员
func (this *AdminDAO) ListEnabledAdmins(tx *dbs.Tx, offset int64, size int64) (result []*Admin, err error) {
_, err = this.Query(tx).
State(AdminStateEnabled).
@@ -253,3 +253,11 @@ func (this *AdminDAO) ListEnabledAdmins(tx *dbs.Tx, offset int64, size int64) (r
FindAll()
return
}
// UpdateAdminTheme 设置管理员Theme
func (this *AdminDAO) UpdateAdminTheme(tx *dbs.Tx, adminId int64, theme string) error {
return this.Query(tx).
Pk(adminId).
Set("theme", theme).
UpdateQuickly()
}

View File

@@ -1,6 +1,6 @@
package models
// 管理员
// Admin 管理员
type Admin struct {
Id uint32 `field:"id"` // ID
IsOn uint8 `field:"isOn"` // 是否启用
@@ -13,6 +13,7 @@ type Admin struct {
State uint8 `field:"state"` // 状态
Modules string `field:"modules"` // 允许的模块
CanLogin uint8 `field:"canLogin"` // 是否可以登录
Theme string `field:"theme"` // 模板设置
}
type AdminOperator struct {
@@ -27,6 +28,7 @@ type AdminOperator struct {
State interface{} // 状态
Modules interface{} // 允许的模块
CanLogin interface{} // 是否可以登录
Theme interface{} // 模板设置
}
func NewAdminOperator() *AdminOperator {

View File

@@ -4,9 +4,10 @@ import (
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
)
// 解析HTTP配置
// DecodeHTTP 解析HTTP配置
func (this *APINode) DecodeHTTP() (*serverconfigs.HTTPProtocolConfig, error) {
if !IsNotNull(this.Http) {
return nil, nil
@@ -25,8 +26,12 @@ func (this *APINode) DecodeHTTP() (*serverconfigs.HTTPProtocolConfig, error) {
return config, nil
}
// 解析HTTPS配置
func (this *APINode) DecodeHTTPS(tx *dbs.Tx) (*serverconfigs.HTTPSProtocolConfig, error) {
// DecodeHTTPS 解析HTTPS配置
func (this *APINode) DecodeHTTPS(tx *dbs.Tx, cacheMap maps.Map) (*serverconfigs.HTTPSProtocolConfig, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
if !IsNotNull(this.Https) {
return nil, nil
}
@@ -44,7 +49,7 @@ func (this *APINode) DecodeHTTPS(tx *dbs.Tx) (*serverconfigs.HTTPSProtocolConfig
if config.SSLPolicyRef != nil {
policyId := config.SSLPolicyRef.SSLPolicyId
if policyId > 0 {
sslPolicy, err := SharedSSLPolicyDAO.ComposePolicyConfig(tx, policyId)
sslPolicy, err := SharedSSLPolicyDAO.ComposePolicyConfig(tx, policyId, cacheMap)
if err != nil {
return nil, err
}
@@ -62,7 +67,7 @@ func (this *APINode) DecodeHTTPS(tx *dbs.Tx) (*serverconfigs.HTTPSProtocolConfig
return config, nil
}
// 解析访问地址
// DecodeAccessAddrs 解析访问地址
func (this *APINode) DecodeAccessAddrs() ([]*serverconfigs.NetworkAddressConfig, error) {
if !IsNotNull(this.AccessAddrs) {
return nil, nil
@@ -82,7 +87,7 @@ func (this *APINode) DecodeAccessAddrs() ([]*serverconfigs.NetworkAddressConfig,
return addrConfigs, nil
}
// 解析访问地址,并返回字符串形式
// DecodeAccessAddrStrings 解析访问地址,并返回字符串形式
func (this *APINode) DecodeAccessAddrStrings() ([]string, error) {
addrs, err := this.DecodeAccessAddrs()
if err != nil {
@@ -95,7 +100,7 @@ func (this *APINode) DecodeAccessAddrStrings() ([]string, error) {
return result, nil
}
// 解析Rest HTTP配置
// DecodeRestHTTP 解析Rest HTTP配置
func (this *APINode) DecodeRestHTTP() (*serverconfigs.HTTPProtocolConfig, error) {
if this.RestIsOn != 1 {
return nil, nil
@@ -117,8 +122,11 @@ func (this *APINode) DecodeRestHTTP() (*serverconfigs.HTTPProtocolConfig, error)
return config, nil
}
// 解析HTTPS配置
func (this *APINode) DecodeRestHTTPS(tx *dbs.Tx) (*serverconfigs.HTTPSProtocolConfig, error) {
// DecodeRestHTTPS 解析HTTPS配置
func (this *APINode) DecodeRestHTTPS(tx *dbs.Tx, cacheMap maps.Map) (*serverconfigs.HTTPSProtocolConfig, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
if this.RestIsOn != 1 {
return nil, nil
}
@@ -139,7 +147,7 @@ func (this *APINode) DecodeRestHTTPS(tx *dbs.Tx) (*serverconfigs.HTTPSProtocolCo
if config.SSLPolicyRef != nil {
policyId := config.SSLPolicyRef.SSLPolicyId
if policyId > 0 {
sslPolicy, err := SharedSSLPolicyDAO.ComposePolicyConfig(tx, policyId)
sslPolicy, err := SharedSSLPolicyDAO.ComposePolicyConfig(tx, policyId, cacheMap)
if err != nil {
return nil, err
}

View File

@@ -120,3 +120,13 @@ func (this *ApiTokenDAO) CreateAPIToken(tx *dbs.Tx, nodeId string, secret string
err := this.Save(tx, op)
return err
}
// FindAllEnabledAPITokens 读取API令牌
func (this *ApiTokenDAO) FindAllEnabledAPITokens(tx *dbs.Tx, role string) (result []*ApiToken, err error) {
_, err = this.Query(tx).
Attr("role", role).
State(ApiTokenStateEnabled).
Slice(&result).
FindAll()
return
}

View File

@@ -104,6 +104,11 @@ func (this *ClientBrowserDAO) FindBrowserIdWithNameCacheable(tx *dbs.Tx, browser
// CreateBrowser 创建浏览器
func (this *ClientBrowserDAO) CreateBrowser(tx *dbs.Tx, browserName string) (int64, error) {
var maxlength = 50
if len(browserName) > maxlength {
browserName = browserName[:50]
}
SharedCacheLocker.Lock()
defer SharedCacheLocker.Unlock()

View File

@@ -104,6 +104,11 @@ func (this *ClientSystemDAO) FindSystemIdWithNameCacheable(tx *dbs.Tx, systemNam
// CreateSystem 创建浏览器
func (this *ClientSystemDAO) CreateSystem(tx *dbs.Tx, systemName string) (int64, error) {
var maxlength = 50
if len(systemName) > maxlength {
systemName = systemName[:50]
}
SharedCacheLocker.Lock()
defer SharedCacheLocker.Unlock()

View File

@@ -176,3 +176,15 @@ func (this *DBNodeDAO) DecodePassword(password string) string {
}
return string(encrypt.MagicKeyDecode(data))
}
// CheckNodeIsOn 检查节点是否已经启用
func (this *DBNodeDAO) CheckNodeIsOn(tx *dbs.Tx, nodeId int64) (bool, error) {
isOn, err := this.Query(tx).
Pk(nodeId).
Result("isOn").
FindIntCol(0)
if err != nil {
return false, err
}
return isOn == 1, nil
}

View File

@@ -3,10 +3,10 @@ package models
import (
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
timeutil "github.com/iwind/TeaGo/utils/time"
"hash/crc32"
"regexp"
@@ -19,9 +19,16 @@ import (
var accessLogDBMapping = map[int64]*dbs.DB{} // dbNodeId => DB
var accessLogLocker = &sync.RWMutex{}
type httpAccessLogDefinition struct {
Name string
HasRemoteAddr bool
HasDomain bool
Exists bool
}
// HTTP服务访问
var httpAccessLogDAOMapping = map[int64]*HTTPAccessLogDAOWrapper{} // dbNodeId => DAO
var httpAccessLogTableMapping = map[string]bool{} // tableName_crc(dsn) => true
var httpAccessLogDAOMapping = map[int64]*HTTPAccessLogDAOWrapper{} // dbNodeId => DAO
var httpAccessLogTableMapping = map[string]*httpAccessLogDefinition{} // tableName_crc(dsn) => true
// DNS服务访问
var nsAccessLogDAOMapping = map[int64]*NSAccessLogDAOWrapper{} // dbNodeId => DAO
@@ -76,7 +83,7 @@ func randomNSAccessLogDAO() (dao *NSAccessLogDAOWrapper) {
}
// 检查表格是否存在
func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) {
func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, hasRemoteAddr bool, hasDomain bool, ok bool, err error) {
if !regexp.MustCompile(`^\d{8}$`).MatchString(day) {
err = errors.New("invalid day '" + day + "', should be YYYYMMDD")
return
@@ -84,25 +91,25 @@ func findHTTPAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bo
config, err := db.Config()
if err != nil {
return "", false, err
return "", false, false, false, err
}
tableName = "edgeHTTPAccessLogs_" + day
cacheKey := tableName + "_" + fmt.Sprintf("%d", crc32.ChecksumIEEE([]byte(config.Dsn)))
accessLogLocker.RLock()
_, ok = httpAccessLogTableMapping[cacheKey]
def, ok := httpAccessLogTableMapping[cacheKey]
accessLogLocker.RUnlock()
if ok {
return tableName, true, nil
return tableName, def.HasRemoteAddr, def.HasDomain, true, nil
}
tableNames, err := db.TableNames()
def, err = findHTTPAccessLogTable(db, day, false)
if err != nil {
return tableName, false, err
return tableName, false, false, false, err
}
return tableName, lists.ContainsString(tableNames, tableName), nil
return tableName, def.HasRemoteAddr, def.HasDomain, def.Exists, nil
}
func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool, err error) {
@@ -135,10 +142,10 @@ func findNSAccessLogTableName(db *dbs.DB, day string) (tableName string, ok bool
}
// 根据日期获取表名
func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
config, err := db.Config()
if err != nil {
return "", err
return nil, err
}
tableName := "edgeHTTPAccessLogs_" + day
@@ -146,36 +153,61 @@ func findHTTPAccessLogTable(db *dbs.DB, day string, force bool) (string, error)
if !force {
accessLogLocker.RLock()
_, ok := httpAccessLogTableMapping[cacheKey]
definition, ok := httpAccessLogTableMapping[cacheKey]
accessLogLocker.RUnlock()
if ok {
return tableName, nil
return definition, nil
}
}
tableNames, err := db.TableNames()
if err != nil {
return tableName, err
return nil, err
}
if lists.ContainsString(tableNames, tableName) {
table, err := db.FindTable(tableName)
if err != nil {
return nil, err
}
accessLogLocker.Lock()
httpAccessLogTableMapping[cacheKey] = true
var definition = &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: table.FindFieldWithName("remoteAddr") != nil,
HasDomain: table.FindFieldWithName("domain") != nil,
Exists: true,
}
httpAccessLogTableMapping[cacheKey] = definition
accessLogLocker.Unlock()
return tableName, nil
return definition, nil
}
if !force {
return &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: true,
HasDomain: true,
Exists: false,
}, nil
}
// 创建表格
_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `serverId` int(11) unsigned DEFAULT '0' COMMENT '服务ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `status` int(3) unsigned DEFAULT '0' COMMENT '状态码',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `content` json DEFAULT NULL COMMENT '日志内容',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `firewallPolicyId` int(11) unsigned DEFAULT '0' COMMENT 'WAF策略ID',\n `firewallRuleGroupId` int(11) unsigned DEFAULT '0' COMMENT 'WAF分组ID',\n `firewallRuleSetId` int(11) unsigned DEFAULT '0' COMMENT 'WAF集ID',\n `firewallRuleId` int(11) unsigned DEFAULT '0' COMMENT 'WAF规则ID',\n `remoteAddr` varchar(64) DEFAULT NULL COMMENT 'IP地址',\n `domain` varchar(128) DEFAULT NULL COMMENT '域名',\n `requestBody` blob COMMENT '请求内容',\n `responseBody` blob COMMENT '响应内容',\n PRIMARY KEY (`id`),\n KEY `serverId` (`serverId`),\n KEY `nodeId` (`nodeId`),\n KEY `serverId_status` (`serverId`,`status`),\n KEY `requestId` (`requestId`),\n KEY `firewallPolicyId` (`firewallPolicyId`),\n KEY `firewallRuleGroupId` (`firewallRuleGroupId`),\n KEY `firewallRuleSetId` (`firewallRuleSetId`),\n KEY `firewallRuleId` (`firewallRuleId`),\n KEY `remoteAddr` (`remoteAddr`),\n KEY `domain` (`domain`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='访问日志';")
if err != nil {
return tableName, err
return nil, err
}
accessLogLocker.Lock()
httpAccessLogTableMapping[cacheKey] = true
var definition = &httpAccessLogDefinition{
Name: tableName,
HasRemoteAddr: true,
Exists: true,
}
httpAccessLogTableMapping[cacheKey] = definition
accessLogLocker.Unlock()
return tableName, nil
return definition, nil
}
func findNSAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
@@ -209,7 +241,7 @@ func findNSAccessLogTable(db *dbs.DB, day string, force bool) (string, error) {
}
// 创建表格
_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `domainId` int(11) unsigned DEFAULT '0' COMMENT '域名ID',\n `recordId` int(11) unsigned DEFAULT '0' COMMENT '记录ID',\n `content` json DEFAULT NULL COMMENT '访问数据',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n PRIMARY KEY (`id`),\n KEY `nodeId` (`nodeId`),\n KEY `domainId` (`domainId`),\n KEY `recordId` (`recordId`),\n KEY `requestId` (`requestId`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='域名服务访问日志';")
_, err = db.Exec("CREATE TABLE `" + tableName + "` (\n `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',\n `nodeId` int(11) unsigned DEFAULT '0' COMMENT '节点ID',\n `domainId` int(11) unsigned DEFAULT '0' COMMENT '域名ID',\n `recordId` int(11) unsigned DEFAULT '0' COMMENT '记录ID',\n `content` json DEFAULT NULL COMMENT '访问数据',\n `requestId` varchar(128) DEFAULT NULL COMMENT '请求ID',\n `createdAt` bigint(11) unsigned DEFAULT '0' COMMENT '创建时间',\n `remoteAddr` varchar(128) DEFAULT NULL COMMENT 'IP',\n PRIMARY KEY (`id`),\n KEY `nodeId` (`nodeId`),\n KEY `domainId` (`domainId`),\n KEY `recordId` (`recordId`),\n KEY `requestId` (`requestId`),\n KEY `remoteAddr` (`remoteAddr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='域名服务访问日志';")
if err != nil {
return tableName, err
}
@@ -234,7 +266,7 @@ func (this *DBNodeInitializer) Start() {
// 初始运行
err := this.loop()
if err != nil {
logs.Println("[DB_NODE]" + err.Error())
remotelogs.Error("DB_NODE", err.Error())
}
// 定时运行
@@ -242,7 +274,7 @@ func (this *DBNodeInitializer) Start() {
for range ticker.C {
err := this.loop()
if err != nil {
logs.Println("[DB_NODE]" + err.Error())
remotelogs.Error("DB_NODE", err.Error())
}
}
}
@@ -268,7 +300,7 @@ func (this *DBNodeInitializer) loop() error {
delete(accessLogDBMapping, nodeId)
delete(httpAccessLogDAOMapping, nodeId)
delete(nsAccessLogDAOMapping, nodeId)
logs.Println("[DB_NODE]close db node '" + strconv.FormatInt(nodeId, 10) + "'")
remotelogs.Error("DB_NODE", "close db node '"+strconv.FormatInt(nodeId, 10)+"'")
}
}
accessLogLocker.Unlock()
@@ -289,7 +321,7 @@ func (this *DBNodeInitializer) loop() error {
// 检查配置是否有变化
oldConfig, err := db.Config()
if err != nil {
logs.Println("[DB_NODE]read database old config failed: " + err.Error())
remotelogs.Error("DB_NODE", "read database old config failed: "+err.Error())
continue
}
@@ -308,22 +340,22 @@ func (this *DBNodeInitializer) loop() error {
}
db, err := dbs.NewInstanceFromConfig(config)
if err != nil {
logs.Println("[DB_NODE]initialize database config failed: " + err.Error())
remotelogs.Error("DB_NODE", "initialize database config failed: "+err.Error())
continue
}
// 检查表是否存在
// httpAccessLog
{
tableName, err := findHTTPAccessLogTable(db, timeutil.Format("Ymd"), false)
tableDef, err := findHTTPAccessLogTable(db, timeutil.Format("Ymd"), true)
if err != nil {
if !strings.Contains(err.Error(), "1050") { // 非表格已存在错误
logs.Println("[DB_NODE]create first table in database node failed: " + err.Error())
remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
// 创建节点日志
createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix())
createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix())
if createLogErr != nil {
logs.Println("[NODE_LOG]" + createLogErr.Error())
remotelogs.Error("NODE_LOG", createLogErr.Error())
}
continue
@@ -335,13 +367,13 @@ func (this *DBNodeInitializer) loop() error {
daoObject := dbs.DAOObject{
Instance: db,
DB: node.Name + "(id:" + strconv.Itoa(int(node.Id)) + ")",
Table: tableName,
Table: tableDef.Name,
PkName: "id",
Model: new(HTTPAccessLog),
}
err = daoObject.Init()
if err != nil {
logs.Println("[DB_NODE]initialize dao failed: " + err.Error())
remotelogs.Error("DB_NODE", "initialize dao failed: "+err.Error())
continue
}
@@ -357,18 +389,17 @@ func (this *DBNodeInitializer) loop() error {
accessLogLocker.Unlock()
}
// nsAccessLog
{
tableName, err := findNSAccessLogTable(db, timeutil.Format("Ymd"), false)
if err != nil {
if !strings.Contains(err.Error(), "1050") { // 非表格已存在错误
logs.Println("[DB_NODE]create first table in database node failed: " + err.Error())
remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
// 创建节点日志
createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix())
createLogErr := SharedNodeLogDAO.CreateLog(nil, nodeconfigs.NodeRoleDatabase, nodeId, 0, 0, "error", "ACCESS_LOG", "can not create access log table: "+err.Error(), time.Now().Unix())
if createLogErr != nil {
logs.Println("[NODE_LOG]" + createLogErr.Error())
remotelogs.Error("NODE_LOG", createLogErr.Error())
}
continue
@@ -386,7 +417,7 @@ func (this *DBNodeInitializer) loop() error {
}
err = daoObject.Init()
if err != nil {
logs.Println("[DB_NODE]initialize dao failed: " + err.Error())
remotelogs.Error("DB_NODE", "initialize dao failed: "+err.Error())
continue
}

View File

@@ -58,14 +58,24 @@ func (this *DNSDomainDAO) DisableDNSDomain(tx *dbs.Tx, id int64) error {
}
// FindEnabledDNSDomain 查找启用中的条目
func (this *DNSDomainDAO) FindEnabledDNSDomain(tx *dbs.Tx, id int64) (*DNSDomain, error) {
func (this *DNSDomainDAO) FindEnabledDNSDomain(tx *dbs.Tx, domainId int64, cacheMap maps.Map) (*DNSDomain, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":record:" + types.String(domainId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*DNSDomain), nil
}
result, err := this.Query(tx).
Pk(id).
Pk(domainId).
Attr("state", DNSDomainStateEnabled).
Find()
if result == nil {
return nil, err
}
cacheMap[cacheKey] = result
return result.(*DNSDomain), err
}
@@ -86,6 +96,7 @@ func (this *DNSDomainDAO) CreateDomain(tx *dbs.Tx, adminId int64, userId int64,
op.Name = name
op.State = DNSDomainStateEnabled
op.IsOn = true
op.IsUp = true
err := this.Save(tx, op)
if err != nil {
return 0, err
@@ -203,7 +214,7 @@ func (this *DNSDomainDAO) FindDomainRouteName(tx *dbs.Tx, domainId int64, routeC
// ExistAvailableDomains 判断是否有域名可选
func (this *DNSDomainDAO) ExistAvailableDomains(tx *dbs.Tx) (bool, error) {
subQuery, err := SharedDNSProviderDAO.Query(tx).
Where("state=1"). // 这里要使用非变量
Where("state=1"). // 这里要使用非变量
ResultPk().
AsSQL()
if err != nil {
@@ -218,6 +229,8 @@ func (this *DNSDomainDAO) ExistAvailableDomains(tx *dbs.Tx) (bool, error) {
// ExistDomainRecord 检查域名解析记录是否存在
func (this *DNSDomainDAO) ExistDomainRecord(tx *dbs.Tx, domainId int64, recordName string, recordType string, recordRoute string, recordValue string) (bool, error) {
recordType = strings.ToUpper(recordType)
query := maps.Map{
"name": recordName,
"type": recordType,
@@ -239,10 +252,31 @@ func (this *DNSDomainDAO) ExistDomainRecord(tx *dbs.Tx, domainId int64, recordNa
}
}
}
recordType = strings.ToUpper(recordType)
return this.Query(tx).
Pk(domainId).
Where("JSON_CONTAINS(records, :query)").
Param("query", query.AsJSON()).
Exist()
}
// FindEnabledDomainWithName 根据名称查找某个域名
func (this *DNSDomainDAO) FindEnabledDomainWithName(tx *dbs.Tx, providerId int64, domainName string) (*DNSDomain, error) {
one, err := this.Query(tx).
State(DNSDomainStateEnabled).
Attr("isOn", true).
Attr("providerId", providerId).
Attr("name", domainName).
Find()
if one != nil {
return one.(*DNSDomain), nil
}
return nil, err
}
// UpdateDomainIsUp 设置是否在线
func (this *DNSDomainDAO) UpdateDomainIsUp(tx *dbs.Tx, domainId int64, isUp bool) error {
return this.Query(tx).
Pk(domainId).
Set("isUp", isUp).
UpdateQuickly()
}

View File

@@ -1,6 +1,6 @@
package dns
// 管理的域名
// DNSDomain 管理的域名
type DNSDomain struct {
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
@@ -14,6 +14,7 @@ type DNSDomain struct {
Data string `field:"data"` // 原始数据信息
Records string `field:"records"` // 所有解析记录
Routes string `field:"routes"` // 线路数据
IsUp uint8 `field:"isUp"` // 是否在线
State uint8 `field:"state"` // 状态
}
@@ -30,6 +31,7 @@ type DNSDomainOperator struct {
Data interface{} // 原始数据信息
Records interface{} // 所有解析记录
Routes interface{} // 线路数据
IsUp interface{} // 是否在线
State interface{} // 状态
}

View File

@@ -36,7 +36,7 @@ func init() {
})
}
// 启用条目
// EnableDNSProvider 启用条目
func (this *DNSProviderDAO) EnableDNSProvider(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -45,7 +45,7 @@ func (this *DNSProviderDAO) EnableDNSProvider(tx *dbs.Tx, id int64) error {
return err
}
// 禁用条目
// DisableDNSProvider 禁用条目
func (this *DNSProviderDAO) DisableDNSProvider(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -54,7 +54,7 @@ func (this *DNSProviderDAO) DisableDNSProvider(tx *dbs.Tx, id int64) error {
return err
}
// 查找启用中的条目
// FindEnabledDNSProvider 查找启用中的条目
func (this *DNSProviderDAO) FindEnabledDNSProvider(tx *dbs.Tx, id int64) (*DNSProvider, error) {
result, err := this.Query(tx).
Pk(id).
@@ -66,7 +66,7 @@ func (this *DNSProviderDAO) FindEnabledDNSProvider(tx *dbs.Tx, id int64) (*DNSPr
return result.(*DNSProvider), err
}
// 创建服务商
// CreateDNSProvider 创建服务商
func (this *DNSProviderDAO) CreateDNSProvider(tx *dbs.Tx, adminId int64, userId int64, providerType string, name string, apiParamsJSON []byte) (int64, error) {
op := NewDNSProviderOperator()
op.AdminId = adminId
@@ -84,7 +84,7 @@ func (this *DNSProviderDAO) CreateDNSProvider(tx *dbs.Tx, adminId int64, userId
return types.Int64(op.Id), nil
}
// 修改服务商
// UpdateDNSProvider 修改服务商
func (this *DNSProviderDAO) UpdateDNSProvider(tx *dbs.Tx, dnsProviderId int64, name string, apiParamsJSON []byte) error {
if dnsProviderId <= 0 {
return errors.New("invalid dnsProviderId")
@@ -106,16 +106,25 @@ func (this *DNSProviderDAO) UpdateDNSProvider(tx *dbs.Tx, dnsProviderId int64, n
return nil
}
// 计算服务商数量
func (this *DNSProviderDAO) CountAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64) (int64, error) {
return dbutils.NewQuery(tx, this, adminId, userId).
State(DNSProviderStateEnabled).
// CountAllEnabledDNSProviders 计算服务商数量
func (this *DNSProviderDAO) CountAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string) (int64, error) {
var query = dbutils.NewQuery(tx, this, adminId, userId)
if len(keyword) > 0 {
query.Where("(name LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
}
return query.State(DNSProviderStateEnabled).
Count()
}
// 列出单页服务商
func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, offset int64, size int64) (result []*DNSProvider, err error) {
_, err = dbutils.NewQuery(tx, this, adminId, userId).
// ListEnabledDNSProviders 列出单页服务商
func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, offset int64, size int64) (result []*DNSProvider, err error) {
var query = dbutils.NewQuery(tx, this, adminId, userId)
if len(keyword) > 0 {
query.Where("(name LIKE :keyword)").
Param("keyword", "%"+keyword+"%")
}
_, err = query.
State(DNSProviderStateEnabled).
Offset(offset).
Limit(size).
@@ -125,7 +134,7 @@ func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, u
return
}
// 列出所有服务商
// FindAllEnabledDNSProviders 列出所有服务商
func (this *DNSProviderDAO) FindAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64) (result []*DNSProvider, err error) {
_, err = dbutils.NewQuery(tx, this, adminId, userId).
State(DNSProviderStateEnabled).
@@ -135,7 +144,7 @@ func (this *DNSProviderDAO) FindAllEnabledDNSProviders(tx *dbs.Tx, adminId int64
return
}
// 查询某个类型下的所有服务商
// FindAllEnabledDNSProvidersWithType 查询某个类型下的所有服务商
func (this *DNSProviderDAO) FindAllEnabledDNSProvidersWithType(tx *dbs.Tx, providerType string) (result []*DNSProvider, err error) {
_, err = this.Query(tx).
State(DNSProviderStateEnabled).
@@ -146,7 +155,7 @@ func (this *DNSProviderDAO) FindAllEnabledDNSProvidersWithType(tx *dbs.Tx, provi
return
}
// 更新数据更新时间
// UpdateProviderDataUpdatedTime 更新数据更新时间
func (this *DNSProviderDAO) UpdateProviderDataUpdatedTime(tx *dbs.Tx, providerId int64) error {
_, err := this.Query(tx).
Pk(providerId).

View File

@@ -94,8 +94,12 @@ func (this *DNSTaskDAO) FindAllDoingTasks(tx *dbs.Tx) (result []*DNSTask, err er
}
// FindAllDoingOrErrorTasks 查找正在执行的和错误的任务
func (this *DNSTaskDAO) FindAllDoingOrErrorTasks(tx *dbs.Tx) (result []*DNSTask, err error) {
_, err = this.Query(tx).
func (this *DNSTaskDAO) FindAllDoingOrErrorTasks(tx *dbs.Tx, nodeClusterId int64) (result []*DNSTask, err error) {
var query = this.Query(tx)
if nodeClusterId > 0 {
query.Attr("clusterId", nodeClusterId)
}
_, err = query.
Where("(isDone=0 OR (isDone=1 AND isOk=0))").
AscPk().
Slice(&result).

View File

@@ -0,0 +1,207 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package dnsutils
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
"github.com/TeaOSLab/EdgeAPI/internal/dnsclients"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/iwind/TeaGo/dbs"
)
// CheckClusterDNS 检查集群的DNS问题
// 藏这么深是避免package循环引用的问题
func CheckClusterDNS(tx *dbs.Tx, cluster *models.NodeCluster) (issues []*pb.DNSIssue, err error) {
clusterId := int64(cluster.Id)
domainId := int64(cluster.DnsDomainId)
// 检查域名
domain, err := dns.SharedDNSDomainDAO.FindEnabledDNSDomain(tx, domainId, nil)
if err != nil {
return nil, err
}
if domain == nil {
issues = append(issues, &pb.DNSIssue{
Target: cluster.Name,
TargetId: clusterId,
Type: "cluster",
Description: "域名选择错误,需要重新选择",
Params: nil,
MustFix: true,
})
return
}
// Provider
provider, err := dns.SharedDNSProviderDAO.FindEnabledDNSProvider(tx, int64(domain.ProviderId))
if err != nil {
return nil, err
}
if provider == nil {
issues = append(issues, &pb.DNSIssue{
Target: cluster.Name,
TargetId: clusterId,
Type: "cluster",
Description: "域名服务商不可用,需要重新选择",
Params: nil,
MustFix: true,
})
return
}
paramsMap, err := provider.DecodeAPIParams()
if err != nil {
issues = append(issues, &pb.DNSIssue{
Target: cluster.Name,
TargetId: clusterId,
Type: "cluster",
Description: "域名服务商参数配置错误,需要重新配置",
Params: nil,
MustFix: true,
})
return
}
var dnsProvider = dnsclients.FindProvider(provider.Type)
if dnsProvider == nil {
issues = append(issues, &pb.DNSIssue{
Target: cluster.Name,
TargetId: clusterId,
Type: "cluster",
Description: "目前不支持\"" + provider.Type + "\"服务商,需要重新配置",
Params: nil,
MustFix: true,
})
return
}
err = dnsProvider.Auth(paramsMap)
if err != nil {
return
}
var defaultRoute = dnsProvider.DefaultRoute()
var hasDefaultRoute = len(defaultRoute) > 0
// 检查二级域名
if len(cluster.DnsName) == 0 {
issues = append(issues, &pb.DNSIssue{
Target: cluster.Name,
TargetId: clusterId,
Type: "cluster",
Description: "没有设置二级域名",
Params: nil,
MustFix: true,
})
return
}
// TODO 检查域名格式
// TODO 检查域名是否已解析
// 检查节点
nodes, err := models.SharedNodeDAO.FindAllEnabledNodesDNSWithClusterId(tx, clusterId, true)
if err != nil {
return nil, err
}
// TODO 检查节点数量不能为0
for _, node := range nodes {
nodeId := int64(node.Id)
routeCodes, err := node.DNSRouteCodesForDomainId(domainId)
if err != nil {
return nil, err
}
if len(routeCodes) == 0 && !hasDefaultRoute {
issues = append(issues, &pb.DNSIssue{
Target: node.Name,
TargetId: nodeId,
Type: "node",
Description: "没有选择节点所属线路",
Params: map[string]string{
"clusterName": cluster.Name,
"clusterId": numberutils.FormatInt64(clusterId),
},
MustFix: true,
})
continue
}
// 检查线路是否在已有线路中
for _, routeCode := range routeCodes {
routeOk, err := domain.ContainsRouteCode(routeCode)
if err != nil {
return nil, err
}
if !routeOk {
issues = append(issues, &pb.DNSIssue{
Target: node.Name,
TargetId: nodeId,
Type: "node",
Description: "线路已经失效,请重新选择",
Params: map[string]string{
"clusterName": cluster.Name,
"clusterId": numberutils.FormatInt64(clusterId),
},
MustFix: true,
})
continue
}
}
// 检查IP地址
ipAddr, err := models.SharedNodeIPAddressDAO.FindFirstNodeAccessIPAddress(tx, nodeId, nodeconfigs.NodeRoleNode)
if err != nil {
return nil, err
}
if len(ipAddr) == 0 {
issues = append(issues, &pb.DNSIssue{
Target: node.Name,
TargetId: nodeId,
Type: "node",
Description: "没有设置IP地址",
Params: map[string]string{
"clusterName": cluster.Name,
"clusterId": numberutils.FormatInt64(clusterId),
},
MustFix: true,
})
continue
}
// TODO 检查是否有解析记录
}
return
}
// FindDefaultDomainRoute 获取域名默认的线路
func FindDefaultDomainRoute(tx *dbs.Tx, domain *dns.DNSDomain) (string, error) {
if domain == nil {
return "", errors.New("can not find domain")
}
provider, err := dns.SharedDNSProviderDAO.FindEnabledDNSProvider(tx, int64(domain.ProviderId))
if err != nil {
return "", err
}
if provider == nil {
return "", errors.New("provider not found")
}
paramsMap, err := provider.DecodeAPIParams()
if err != nil {
return "", errors.New("decode provider params failed: " + err.Error())
}
var dnsProvider = dnsclients.FindProvider(provider.Type)
if dnsProvider == nil {
return "", errors.New("not supported provider type '" + provider.Type + "'")
}
err = dnsProvider.Auth(paramsMap)
if err != nil {
return "", err
}
return dnsProvider.DefaultRoute(), nil
}

View File

@@ -0,0 +1,29 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package dnsutils
import (
"github.com/TeaOSLab/EdgeAPI/internal/db/models"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/logs"
"testing"
)
func TestNodeClusterDAO_CheckClusterDNS(t *testing.T) {
dbs.NotifyReady()
var tx *dbs.Tx
cluster, err := models.SharedNodeClusterDAO.FindEnabledNodeCluster(tx, 34)
if err != nil {
t.Fatal(err)
}
if cluster == nil {
t.Log("cluster not found, skip the test")
return
}
issues, err := CheckClusterDNS(tx, cluster)
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(issues, t)
}

View File

@@ -4,7 +4,9 @@ import (
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/configs"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
@@ -12,6 +14,7 @@ import (
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"net"
"net/http"
"regexp"
"sort"
@@ -69,7 +72,7 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
for _, accessLog := range accessLogs {
day := timeutil.Format("Ymd", time.Unix(accessLog.Timestamp, 0))
table, err := findHTTPAccessLogTable(dao.Instance, day, false)
tableDef, err := findHTTPAccessLogTable(dao.Instance, day, false)
if err != nil {
return err
}
@@ -85,6 +88,14 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
fields["firewallRuleSetId"] = accessLog.FirewallRuleSetId
fields["firewallRuleId"] = accessLog.FirewallRuleId
// TODO 根据集群、服务设置获取IP
if tableDef.HasRemoteAddr {
fields["remoteAddr"] = accessLog.RemoteAddr
}
if tableDef.HasDomain {
fields["domain"] = accessLog.Host
}
content, err := json.Marshal(accessLog)
if err != nil {
return err
@@ -92,23 +103,25 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
fields["content"] = content
_, err = dao.Query(tx).
Table(table).
Table(tableDef.Name).
Sets(fields).
Insert()
if err != nil {
// 是否为 Error 1146: Table 'xxx.xxx' doesn't exist 如果是,则创建表之后重试
if strings.Contains(err.Error(), "1146") {
table, err = findHTTPAccessLogTable(dao.Instance, day, true)
tableDef, err = findHTTPAccessLogTable(dao.Instance, day, true)
if err != nil {
return err
}
_, err = dao.Query(tx).
Table(table).
Table(tableDef.Name).
Sets(fields).
Insert()
if err != nil {
return err
}
} else {
logs.Println("HTTP_ACCESS_LOG", err.Error())
}
}
}
@@ -117,7 +130,20 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLogsWithDAO(tx *dbs.Tx, daoWrapper
}
// ListAccessLogs 读取往前的 单页访问日志
func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
size int64,
day string,
serverId int64,
reverse bool,
hasError bool,
firewallPolicyId int64,
firewallRuleGroupId int64,
firewallRuleSetId int64,
hasFirewallPolicy bool,
userId int64,
keyword string,
ip string,
domain string) (result []*HTTPAccessLog, nextLastRequestId string, hasMore bool, err error) {
if len(day) != 8 {
return
}
@@ -127,18 +153,18 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string, s
size = 1000
}
result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword)
result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
if err != nil || int64(len(result)) < size {
return
}
moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword)
moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
hasMore = len(moreResult) > 0
return
}
// 读取往前的单页访问日志
func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, size int64, day string, serverId int64, reverse bool, hasError bool, firewallPolicyId int64, firewallRuleGroupId int64, firewallRuleSetId int64, hasFirewallPolicy bool, userId int64, keyword string, ip string, domain string) (result []*HTTPAccessLog, nextLastRequestId string, err error) {
if size <= 0 {
return nil, lastRequestId, nil
}
@@ -179,7 +205,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
dao := daoWrapper.DAO
tableName, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
tableName, hasRemoteAddrField, hasDomainField, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
if !exists {
// 表格不存在则跳过
return
@@ -212,45 +238,105 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, s
}
if hasFirewallPolicy {
query.Where("firewallPolicyId>0")
query.UseIndex("firewallPolicyId")
}
// keyword
if len(ip) > 0 {
// TODO 支持IP范围
if hasRemoteAddrField {
// IP格式
if strings.Contains(ip, ",") || strings.Contains(ip, "-") {
rangeConfig, err := shared.ParseIPRange(ip)
if err == nil {
if len(rangeConfig.IPFrom) > 0 && len(rangeConfig.IPTo) > 0 {
query.Between("INET_ATON(remoteAddr)", utils.IP2Long(rangeConfig.IPFrom), utils.IP2Long(rangeConfig.IPTo))
}
}
} else {
query.Attr("remoteAddr", ip)
query.UseIndex("remoteAddr")
}
} else {
query.Where("JSON_EXTRACT(content, '$.remoteAddr')=:ip1").
Param("ip1", ip)
}
}
if len(domain) > 0 {
if hasDomainField {
if strings.Contains(domain, "*") {
domain = strings.ReplaceAll(domain, "*", "%")
domain = regexp.MustCompile(`[^a-zA-Z0-9-.%]`).ReplaceAllString(domain, "")
query.Where("domain LIKE :host2").
Param("host2", domain)
} else {
query.Attr("domain", domain)
query.UseIndex("domain")
}
} else {
query.Where("JSON_EXTRACT(content, '$.host')=:host1").
Param("host1", domain)
}
}
if len(keyword) > 0 {
useOriginKeyword := false
// remoteAddr
if hasRemoteAddrField && net.ParseIP(keyword) != nil {
query.Attr("remoteAddr", keyword)
} else if hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
keyword = keyword[3:]
pieces := strings.SplitN(keyword, ",", 2)
if len(pieces) == 1 || len(pieces[1]) == 0 {
query.Attr("remoteAddr", pieces[0])
} else {
query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
}
} else {
if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
keyword = keyword[3:]
}
where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword"
useOriginKeyword := false
// 请求方法
if keyword == http.MethodGet ||
keyword == http.MethodPost ||
keyword == http.MethodHead ||
keyword == http.MethodConnect ||
keyword == http.MethodPut ||
keyword == http.MethodTrace ||
keyword == http.MethodOptions ||
keyword == http.MethodDelete ||
keyword == http.MethodPatch {
where += " OR JSON_EXTRACT(content, '$.requestMethod')=:originKeyword"
useOriginKeyword = true
}
where := "JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.requestURI') LIKE :keyword OR JSON_EXTRACT(content, '$.host') LIKE :keyword OR JSON_EXTRACT(content, '$.userAgent') LIKE :keyword"
// 响应状态码
if regexp.MustCompile(`^\d{3}$`).MatchString(keyword) {
where += " OR JSON_EXTRACT(content, '$.status')=:intKeyword"
query.Param("intKeyword", types.Int(keyword))
}
jsonKeyword, err := json.Marshal(keyword)
if err == nil {
where += " OR JSON_CONTAINS(content, :jsonKeyword, '$.tags')"
query.Param("jsonKeyword", jsonKeyword)
}
if regexp.MustCompile(`^\d{3}-\d{3}$`).MatchString(keyword) {
pieces := strings.Split(keyword, "-")
where += " OR JSON_EXTRACT(content, '$.status') BETWEEN :intKeyword1 AND :intKeyword2"
query.Param("intKeyword1", types.Int(pieces[0]))
query.Param("intKeyword2", types.Int(pieces[1]))
}
// 请求方法
if keyword == http.MethodGet ||
keyword == http.MethodPost ||
keyword == http.MethodHead ||
keyword == http.MethodConnect ||
keyword == http.MethodPut ||
keyword == http.MethodTrace ||
keyword == http.MethodOptions ||
keyword == http.MethodDelete ||
keyword == http.MethodPatch {
where += " OR JSON_EXTRACT(content, '$.requestMethod')=:originKeyword"
useOriginKeyword = true
}
query.Where("("+where+")").
Param("keyword", "%"+keyword+"%")
if useOriginKeyword {
query.Param("originKeyword", keyword)
// 响应状态码
if regexp.MustCompile(`^\d{3}$`).MatchString(keyword) {
where += " OR JSON_EXTRACT(content, '$.status')=:intKeyword"
query.Param("intKeyword", types.Int(keyword))
}
if regexp.MustCompile(`^\d{3}-\d{3}$`).MatchString(keyword) {
pieces := strings.Split(keyword, "-")
where += " OR JSON_EXTRACT(content, '$.status') BETWEEN :intKeyword1 AND :intKeyword2"
query.Param("intKeyword1", types.Int(pieces[0]))
query.Param("intKeyword2", types.Int(pieces[1]))
}
query.Where("("+where+")").
Param("keyword", "%"+keyword+"%")
if useOriginKeyword {
query.Param("originKeyword", keyword)
}
}
}
@@ -350,7 +436,7 @@ func (this *HTTPAccessLogDAO) FindAccessLogWithRequestId(tx *dbs.Tx, requestId s
dao := daoWrapper.DAO
tableName, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
tableName, _, _, exists, err := findHTTPAccessLogTableName(dao.Instance, day)
if err != nil {
logs.Println("[DB_NODE]" + err.Error())
return

View File

@@ -41,7 +41,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
t.Fatal(err)
}
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0, false, 0, "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0, false, 0, "", "", "")
if err != nil {
t.Fatal(err)
}
@@ -68,7 +68,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
times := 0 // 防止循环次数太多
for {
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0, false, 0, "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), 0, false, false, 0, 0, 0, false, 0, "", "", "")
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)
@@ -99,7 +99,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
}
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true, false, 0, 0, 0, false, 0, "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), 0, true, false, 0, 0, 0, false, 0, "", "", "")
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)
@@ -124,7 +124,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
times := 0 // 防止循环次数太多
for {
before := time.Now()
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false, false, 0, 0, 0, false, 0, "")
accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), 0, false, false, 0, 0, 0, false, 0, "", "", "")
cost := time.Since(before).Seconds()
if err != nil {
t.Fatal(err)

View File

@@ -1,6 +1,6 @@
package models
//
// HTTPAccessLog 访问日志
type HTTPAccessLog struct {
Id uint64 `field:"id"` // ID
ServerId uint32 `field:"serverId"` // 服务ID
@@ -13,6 +13,10 @@ type HTTPAccessLog struct {
FirewallRuleGroupId uint32 `field:"firewallRuleGroupId"` // WAF分组ID
FirewallRuleSetId uint32 `field:"firewallRuleSetId"` // WAF集ID
FirewallRuleId uint32 `field:"firewallRuleId"` // WAF规则ID
RemoteAddr string `field:"remoteAddr"` // IP地址
Domain string `field:"domain"` // 域名
RequestBody string `field:"requestBody"` // 请求内容
ResponseBody string `field:"responseBody"` // 响应内容
}
type HTTPAccessLogOperator struct {
@@ -27,6 +31,10 @@ type HTTPAccessLogOperator struct {
FirewallRuleGroupId interface{} // WAF分组ID
FirewallRuleSetId interface{} // WAF集ID
FirewallRuleId interface{} // WAF规则ID
RemoteAddr interface{} // IP地址
Domain interface{} // 域名
RequestBody interface{} // 请求内容
ResponseBody interface{} // 响应内容
}
func NewHTTPAccessLogOperator() *HTTPAccessLogOperator {

View File

@@ -1,12 +1,13 @@
package models
import (
"bytes"
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
)
const (
@@ -35,12 +36,12 @@ func init() {
})
}
// 初始化
// Init 初始化
func (this *HTTPAccessLogPolicyDAO) Init() {
_ = this.DAOObject.Init()
}
// 启用条目
// EnableHTTPAccessLogPolicy 启用条目
func (this *HTTPAccessLogPolicyDAO) EnableHTTPAccessLogPolicy(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -49,7 +50,7 @@ func (this *HTTPAccessLogPolicyDAO) EnableHTTPAccessLogPolicy(tx *dbs.Tx, id int
return err
}
// 禁用条目
// DisableHTTPAccessLogPolicy 禁用条目
func (this *HTTPAccessLogPolicyDAO) DisableHTTPAccessLogPolicy(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -58,7 +59,7 @@ func (this *HTTPAccessLogPolicyDAO) DisableHTTPAccessLogPolicy(tx *dbs.Tx, id in
return err
}
// 查找启用中的条目
// FindEnabledHTTPAccessLogPolicy 查找启用中的条目
func (this *HTTPAccessLogPolicyDAO) FindEnabledHTTPAccessLogPolicy(tx *dbs.Tx, id int64) (*HTTPAccessLogPolicy, error) {
result, err := this.Query(tx).
Pk(id).
@@ -70,7 +71,7 @@ func (this *HTTPAccessLogPolicyDAO) FindEnabledHTTPAccessLogPolicy(tx *dbs.Tx, i
return result.(*HTTPAccessLogPolicy), err
}
// 根据主键查找名称
// FindHTTPAccessLogPolicyName 根据主键查找名称
func (this *HTTPAccessLogPolicyDAO) FindHTTPAccessLogPolicyName(tx *dbs.Tx, id int64) (string, error) {
return this.Query(tx).
Pk(id).
@@ -78,51 +79,116 @@ func (this *HTTPAccessLogPolicyDAO) FindHTTPAccessLogPolicyName(tx *dbs.Tx, id i
FindStringCol("")
}
// 查找所有可用策略信息
func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAccessLogPolicies(tx *dbs.Tx) (result []*HTTPAccessLogPolicy, err error) {
// CountAllEnabledPolicies 计算策略数量
func (this *HTTPAccessLogPolicyDAO) CountAllEnabledPolicies(tx *dbs.Tx) (int64, error) {
return this.Query(tx).
State(HTTPAccessLogPolicyStateEnabled).
Count()
}
// ListEnabledPolicies 查找所有可用策略信息
func (this *HTTPAccessLogPolicyDAO) ListEnabledPolicies(tx *dbs.Tx, offset int64, size int64) (result []*HTTPAccessLogPolicy, err error) {
_, err = this.Query(tx).
State(HTTPAccessLogPolicyStateEnabled).
DescPk().
Offset(offset).
Limit(size).
Slice(&result).
FindAll()
return
}
// 组合配置
func (this *HTTPAccessLogPolicyDAO) ComposeAccessLogPolicyConfig(tx *dbs.Tx, policyId int64) (*serverconfigs.HTTPAccessLogStoragePolicy, error) {
policy, err := this.FindEnabledHTTPAccessLogPolicy(tx, policyId)
if err != nil {
return nil, err
}
if policy == nil {
return nil, nil
}
config := &serverconfigs.HTTPAccessLogStoragePolicy{}
config.Id = int64(policy.Id)
config.IsOn = policy.IsOn == 1
config.Name = policy.Name
config.Type = policy.Type
// 选项
if IsNotNull(policy.Options) {
m := map[string]interface{}{}
err = json.Unmarshal([]byte(policy.Options), &m)
if err != nil {
return nil, err
}
config.Options = m
}
// 条件
if IsNotNull(policy.Conds) {
condsConfig := &shared.HTTPRequestCondsConfig{}
err = json.Unmarshal([]byte(policy.Conds), condsConfig)
if err != nil {
return nil, err
}
config.Conds = condsConfig
}
return config, nil
// FindAllEnabledAndOnPolicies 获取所有的策略信息
func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (result []*HTTPAccessLogPolicy, err error) {
_, err = this.Query(tx).
State(HTTPAccessLogPolicyStateEnabled).
Attr("isOn", true).
Slice(&result).
FindAll()
return
}
// CreatePolicy 创建策略
func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool) (policyId int64, err error) {
var op = NewHTTPAccessLogPolicyOperator()
op.Name = name
op.Type = policyType
if len(optionsJSON) > 0 {
op.Options = optionsJSON
}
if len(condsJSON) > 0 {
op.Conds = condsJSON
}
op.IsPublic = isPublic
op.IsOn = true
op.State = HTTPAccessLogPolicyStateEnabled
return this.SaveInt64(tx, op)
}
// UpdatePolicy 修改策略
func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, isOn bool) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
oldOne, err := this.Query(tx).
Pk(policyId).
Find()
if err != nil {
return err
}
if oldOne == nil {
return nil
}
var oldPolicy = oldOne.(*HTTPAccessLogPolicy)
var op = NewHTTPAccessLogPolicyOperator()
op.Id = policyId
op.Name = name
if len(optionsJSON) > 0 {
op.Options = optionsJSON
} else {
op.Options = "{}"
}
if len(condsJSON) > 0 {
op.Conds = condsJSON
} else {
op.Conds = "{}"
}
// 版本号
if len(oldPolicy.Options) == 0 || len(optionsJSON) == 0 {
op.Version = dbs.SQL("version+1")
} else {
var m1 = maps.Map{}
_ = json.Unmarshal([]byte(oldPolicy.Options), &m1)
var m2 = maps.Map{}
_ = json.Unmarshal(optionsJSON, &m2)
if bytes.Compare(m1.AsJSON(), m2.AsJSON()) != 0 {
op.Version = dbs.SQL("version+1")
}
}
op.IsPublic = isPublic
op.IsOn = isOn
return this.Save(tx, op)
}
// CancelAllPublicPolicies 取消别的公用的策略
func (this *HTTPAccessLogPolicyDAO) CancelAllPublicPolicies(tx *dbs.Tx) error {
return this.Query(tx).
State(HTTPAccessLogPolicyStateEnabled).
Set("isPublic", 0).
UpdateQuickly()
}
// FindCurrentPublicPolicyId 取得当前的公用策略
func (this *HTTPAccessLogPolicyDAO) FindCurrentPublicPolicyId(tx *dbs.Tx) (int64, error) {
return this.Query(tx).
State(HTTPAccessLogPolicyStateEnabled).
Attr("isPublic", 1).
ResultPk().
FindInt64Col(0)
}

View File

@@ -1,6 +1,6 @@
package models
// 访问日志策略
// HTTPAccessLogPolicy 访问日志策略
type HTTPAccessLogPolicy struct {
Id uint32 `field:"id"` // ID
TemplateId uint32 `field:"templateId"` // 模版ID
@@ -13,6 +13,8 @@ type HTTPAccessLogPolicy struct {
Type string `field:"type"` // 存储类型
Options string `field:"options"` // 存储选项
Conds string `field:"conds"` // 请求条件
IsPublic uint8 `field:"isPublic"` // 是否为公用
Version uint32 `field:"version"` // 版本号
}
type HTTPAccessLogPolicyOperator struct {
@@ -27,6 +29,8 @@ type HTTPAccessLogPolicyOperator struct {
Type interface{} // 存储类型
Options interface{} // 存储选项
Conds interface{} // 请求条件
IsPublic interface{} // 是否为公用
Version interface{} // 版本号
}
func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {

View File

@@ -7,6 +7,8 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
const (
@@ -94,7 +96,16 @@ func (this *HTTPAuthPolicyDAO) UpdateHTTPAuthPolicy(tx *dbs.Tx, policyId int64,
}
// ComposePolicyConfig 组合配置
func (this *HTTPAuthPolicyDAO) ComposePolicyConfig(tx *dbs.Tx, policyId int64) (*serverconfigs.HTTPAuthPolicy, error) {
func (this *HTTPAuthPolicyDAO) ComposePolicyConfig(tx *dbs.Tx, policyId int64, cacheMap maps.Map) (*serverconfigs.HTTPAuthPolicy, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(policyId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*serverconfigs.HTTPAuthPolicy), nil
}
policy, err := this.FindEnabledHTTPAuthPolicy(tx, policyId)
if err != nil {
return nil, err
@@ -119,6 +130,8 @@ func (this *HTTPAuthPolicyDAO) ComposePolicyConfig(tx *dbs.Tx, policyId int64) (
}
config.Params = params
cacheMap[cacheKey] = config
return config, nil
}

View File

@@ -8,6 +8,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
@@ -155,6 +156,44 @@ func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name st
return types.Int64(op.Id), nil
}
// CreateDefaultCachePolicy 创建默认的缓存策略
func (this *HTTPCachePolicyDAO) CreateDefaultCachePolicy(tx *dbs.Tx, name string) (int64, error) {
var capacity = &shared.SizeCapacity{
Count: 64,
Unit: shared.SizeCapacityUnitGB,
}
capacityJSON, err := capacity.AsJSON()
if err != nil {
return 0, err
}
var maxSize = &shared.SizeCapacity{
Count: 256,
Unit: shared.SizeCapacityUnitMB,
}
if err != nil {
return 0, err
}
maxSizeJSON, err := maxSize.AsJSON()
if err != nil {
return 0, err
}
var storageOptions = &serverconfigs.HTTPFileCacheStorage{
Dir: "/opt/cache",
}
storageOptionsJSON, err := json.Marshal(storageOptions)
if err != nil {
return 0, err
}
policyId, err := this.CreateCachePolicy(tx, true, "\""+name+"\"缓存策略", "默认创建的缓存策略", capacityJSON, 0, maxSizeJSON, serverconfigs.CachePolicyStorageFile, storageOptionsJSON)
if err != nil {
return 0, err
}
return policyId, nil
}
// UpdateCachePolicy 修改缓存策略
func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte) error {
if policyId <= 0 {
@@ -185,7 +224,16 @@ func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, is
}
// ComposeCachePolicy 组合配置
func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64) (*serverconfigs.HTTPCachePolicy, error) {
func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64, cacheMap maps.Map) (*serverconfigs.HTTPCachePolicy, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(policyId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*serverconfigs.HTTPCachePolicy), nil
}
policy, err := this.FindEnabledHTTPCachePolicy(tx, policyId)
if err != nil {
return nil, err
@@ -243,6 +291,8 @@ func (this *HTTPCachePolicyDAO) ComposeCachePolicy(tx *dbs.Tx, policyId int64) (
config.CacheRefs = refs
}
cacheMap[cacheKey] = config
return config, nil
}
@@ -284,7 +334,7 @@ func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, keyword
cachePolicies := []*serverconfigs.HTTPCachePolicy{}
for _, policyId := range cachePolicyIds {
cachePolicyConfig, err := this.ComposeCachePolicy(tx, policyId)
cachePolicyConfig, err := this.ComposeCachePolicy(tx, policyId, nil)
if err != nil {
return nil, errors.Wrap(err)
}

View File

@@ -7,6 +7,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
@@ -113,8 +114,73 @@ func (this *HTTPFirewallPolicyDAO) CreateFirewallPolicy(tx *dbs.Tx, userId int64
return types.Int64(op.Id), err
}
// CreateDefaultFirewallPolicy 创建默认的WAF策略
func (this *HTTPFirewallPolicyDAO) CreateDefaultFirewallPolicy(tx *dbs.Tx, name string) (int64, error) {
policyId, err := this.CreateFirewallPolicy(tx, 0, 0, true, "\""+name+"\"WAF策略", "默认创建的WAF策略", nil, nil)
if err != nil {
return 0, err
}
// 初始化
var groupCodes = []string{}
templatePolicy := firewallconfigs.HTTPFirewallTemplate()
for _, group := range templatePolicy.AllRuleGroups() {
groupCodes = append(groupCodes, group.Code)
}
inboundConfig := &firewallconfigs.HTTPFirewallInboundConfig{IsOn: true}
outboundConfig := &firewallconfigs.HTTPFirewallOutboundConfig{IsOn: true}
if templatePolicy.Inbound != nil {
for _, group := range templatePolicy.Inbound.Groups {
isOn := lists.ContainsString(groupCodes, group.Code)
group.IsOn = isOn
groupId, err := SharedHTTPFirewallRuleGroupDAO.CreateGroupFromConfig(tx, group)
if err != nil {
return 0, err
}
inboundConfig.GroupRefs = append(inboundConfig.GroupRefs, &firewallconfigs.HTTPFirewallRuleGroupRef{
IsOn: true,
GroupId: groupId,
})
}
}
if templatePolicy.Outbound != nil {
for _, group := range templatePolicy.Outbound.Groups {
isOn := lists.ContainsString(groupCodes, group.Code)
group.IsOn = isOn
groupId, err := SharedHTTPFirewallRuleGroupDAO.CreateGroupFromConfig(tx, group)
if err != nil {
return 0, err
}
outboundConfig.GroupRefs = append(outboundConfig.GroupRefs, &firewallconfigs.HTTPFirewallRuleGroupRef{
IsOn: true,
GroupId: groupId,
})
}
}
inboundConfigJSON, err := json.Marshal(inboundConfig)
if err != nil {
return 0, err
}
outboundConfigJSON, err := json.Marshal(outboundConfig)
if err != nil {
return 0, err
}
err = this.UpdateFirewallPolicyInboundAndOutbound(tx, policyId, inboundConfigJSON, outboundConfigJSON, false)
if err != nil {
return 0, err
}
return policyId, nil
}
// UpdateFirewallPolicyInboundAndOutbound 修改策略的Inbound和Outbound
func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInboundAndOutbound(tx *dbs.Tx, policyId int64, inboundJSON []byte, outboundJSON []byte) error {
func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInboundAndOutbound(tx *dbs.Tx, policyId int64, inboundJSON []byte, outboundJSON []byte, shouldNotify bool) error {
if policyId <= 0 {
return errors.New("invalid policyId")
}
@@ -135,7 +201,11 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInboundAndOutbound(tx *db
return err
}
return this.NotifyUpdate(tx, policyId)
if shouldNotify {
return this.NotifyUpdate(tx, policyId)
}
return nil
}
// UpdateFirewallPolicyInbound 修改策略的Inbound
@@ -223,7 +293,16 @@ func (this *HTTPFirewallPolicyDAO) ListEnabledFirewallPolicies(tx *dbs.Tx, keywo
}
// ComposeFirewallPolicy 组合策略配置
func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId int64) (*firewallconfigs.HTTPFirewallPolicy, error) {
func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId int64, cacheMap maps.Map) (*firewallconfigs.HTTPFirewallPolicy, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(policyId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*firewallconfigs.HTTPFirewallPolicy), nil
}
policy, err := this.FindEnabledHTTPFirewallPolicy(tx, policyId)
if err != nil {
return nil, err
@@ -304,6 +383,8 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
config.BlockOptions = blockAction
}
cacheMap[cacheKey] = config
return config, nil
}

View File

@@ -37,12 +37,12 @@ func init() {
})
}
// 初始化
// Init 初始化
func (this *HTTPFirewallRuleSetDAO) Init() {
_ = this.DAOObject.Init()
}
// 启用条目
// EnableHTTPFirewallRuleSet 启用条目
func (this *HTTPFirewallRuleSetDAO) EnableHTTPFirewallRuleSet(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -51,7 +51,7 @@ func (this *HTTPFirewallRuleSetDAO) EnableHTTPFirewallRuleSet(tx *dbs.Tx, id int
return err
}
// 禁用条目
// DisableHTTPFirewallRuleSet 禁用条目
func (this *HTTPFirewallRuleSetDAO) DisableHTTPFirewallRuleSet(tx *dbs.Tx, ruleSetId int64) error {
_, err := this.Query(tx).
Pk(ruleSetId).
@@ -63,7 +63,7 @@ func (this *HTTPFirewallRuleSetDAO) DisableHTTPFirewallRuleSet(tx *dbs.Tx, ruleS
return this.NotifyUpdate(tx, ruleSetId)
}
// 查找启用中的条目
// FindEnabledHTTPFirewallRuleSet 查找启用中的条目
func (this *HTTPFirewallRuleSetDAO) FindEnabledHTTPFirewallRuleSet(tx *dbs.Tx, id int64) (*HTTPFirewallRuleSet, error) {
result, err := this.Query(tx).
Pk(id).
@@ -75,7 +75,7 @@ func (this *HTTPFirewallRuleSetDAO) FindEnabledHTTPFirewallRuleSet(tx *dbs.Tx, i
return result.(*HTTPFirewallRuleSet), err
}
// 根据主键查找名称
// FindHTTPFirewallRuleSetName 根据主键查找名称
func (this *HTTPFirewallRuleSetDAO) FindHTTPFirewallRuleSetName(tx *dbs.Tx, id int64) (string, error) {
return this.Query(tx).
Pk(id).
@@ -83,7 +83,7 @@ func (this *HTTPFirewallRuleSetDAO) FindHTTPFirewallRuleSetName(tx *dbs.Tx, id i
FindStringCol("")
}
// 组合配置
// ComposeFirewallRuleSet 组合配置
func (this *HTTPFirewallRuleSetDAO) ComposeFirewallRuleSet(tx *dbs.Tx, setId int64) (*firewallconfigs.HTTPFirewallRuleSet, error) {
set, err := this.FindEnabledHTTPFirewallRuleSet(tx, setId)
if err != nil {
@@ -118,20 +118,19 @@ func (this *HTTPFirewallRuleSetDAO) ComposeFirewallRuleSet(tx *dbs.Tx, setId int
}
}
config.Action = set.Action
if IsNotNull(set.ActionOptions) {
options := maps.Map{}
err = json.Unmarshal([]byte(set.ActionOptions), &options)
var actionConfigs = []*firewallconfigs.HTTPFirewallActionConfig{}
if len(set.Actions) > 0 {
err = json.Unmarshal([]byte(set.Actions), &actionConfigs)
if err != nil {
return nil, err
}
config.ActionOptions = options
config.Actions = actionConfigs
}
return config, nil
}
// 从配置中创建规则集
// CreateOrUpdateSetFromConfig 从配置中创建规则集
func (this *HTTPFirewallRuleSetDAO) CreateOrUpdateSetFromConfig(tx *dbs.Tx, setConfig *firewallconfigs.HTTPFirewallRuleSet) (int64, error) {
op := NewHTTPFirewallRuleSetOperator()
op.State = HTTPFirewallRuleSetStateEnabled
@@ -140,19 +139,19 @@ func (this *HTTPFirewallRuleSetDAO) CreateOrUpdateSetFromConfig(tx *dbs.Tx, setC
op.Name = setConfig.Name
op.Description = setConfig.Description
op.Connector = setConfig.Connector
op.Action = setConfig.Action
op.Code = setConfig.Code
if setConfig.ActionOptions != nil {
actionOptionsJSON, err := json.Marshal(setConfig.ActionOptions)
if len(setConfig.Actions) == 0 {
op.Actions = "[]"
} else {
actionsJSON, err := json.Marshal(setConfig.Actions)
if err != nil {
return 0, err
}
op.ActionOptions = actionOptionsJSON
} else {
op.ActionOptions = "{}"
op.Actions = actionsJSON
}
op.Code = setConfig.Code
// rules
ruleRefs := []*firewallconfigs.HTTPFirewallRuleRef{}
for _, ruleConfig := range setConfig.Rules {
@@ -186,7 +185,7 @@ func (this *HTTPFirewallRuleSetDAO) CreateOrUpdateSetFromConfig(tx *dbs.Tx, setC
return types.Int64(op.Id), nil
}
// 设置是否启用
// UpdateRuleSetIsOn 设置是否启用
func (this *HTTPFirewallRuleSetDAO) UpdateRuleSetIsOn(tx *dbs.Tx, ruleSetId int64, isOn bool) error {
if ruleSetId <= 0 {
return errors.New("invalid ruleSetId")
@@ -201,7 +200,7 @@ func (this *HTTPFirewallRuleSetDAO) UpdateRuleSetIsOn(tx *dbs.Tx, ruleSetId int6
return this.NotifyUpdate(tx, ruleSetId)
}
// 根据规则查找规则集
// FindEnabledRuleSetIdWithRuleId 根据规则查找规则集
func (this *HTTPFirewallRuleSetDAO) FindEnabledRuleSetIdWithRuleId(tx *dbs.Tx, ruleId int64) (int64, error) {
return this.Query(tx).
State(HTTPFirewallRuleStateEnabled).
@@ -211,7 +210,7 @@ func (this *HTTPFirewallRuleSetDAO) FindEnabledRuleSetIdWithRuleId(tx *dbs.Tx, r
FindInt64Col(0)
}
// 检查用户
// CheckUserRuleSet 检查用户
func (this *HTTPFirewallRuleSetDAO) CheckUserRuleSet(tx *dbs.Tx, userId int64, setId int64) error {
groupId, err := SharedHTTPFirewallRuleGroupDAO.FindRuleGroupIdWithRuleSetId(tx, setId)
if err != nil {
@@ -223,7 +222,7 @@ func (this *HTTPFirewallRuleSetDAO) CheckUserRuleSet(tx *dbs.Tx, userId int64, s
return SharedHTTPFirewallRuleGroupDAO.CheckUserRuleGroup(tx, userId, groupId)
}
// 通知更新
// NotifyUpdate 通知更新
func (this *HTTPFirewallRuleSetDAO) NotifyUpdate(tx *dbs.Tx, setId int64) error {
groupId, err := SharedHTTPFirewallRuleGroupDAO.FindRuleGroupIdWithRuleSetId(tx, setId)
if err != nil {

View File

@@ -1,6 +1,6 @@
package models
// 防火墙规则集
// HTTPFirewallRuleSet 防火墙规则集
type HTTPFirewallRuleSet struct {
Id uint32 `field:"id"` // ID
IsOn uint8 `field:"isOn"` // 是否启用
@@ -13,8 +13,9 @@ type HTTPFirewallRuleSet struct {
State uint8 `field:"state"` // 状态
AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID
Action string `field:"action"` // 执行的动作
ActionOptions string `field:"actionOptions"` // 动作的选项
Action string `field:"action"` // 执行的动作(过期)
ActionOptions string `field:"actionOptions"` // 动作的选项(过期)
Actions string `field:"actions"` // 一组动作
}
type HTTPFirewallRuleSetOperator struct {
@@ -29,8 +30,9 @@ type HTTPFirewallRuleSetOperator struct {
State interface{} // 状态
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
Action interface{} // 执行的动作
ActionOptions interface{} // 动作的选项
Action interface{} // 执行的动作(过期)
ActionOptions interface{} // 动作的选项(过期)
Actions interface{} // 一组动作
}
func NewHTTPFirewallRuleSetOperator() *HTTPFirewallRuleSetOperator {

View File

@@ -8,6 +8,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
@@ -83,7 +84,7 @@ func (this *HTTPLocationDAO) FindHTTPLocationName(tx *dbs.Tx, id int64) (string,
FindStringCol("")
}
// CreateLocation 创建路规则
// CreateLocation 创建路规则
func (this *HTTPLocationDAO) CreateLocation(tx *dbs.Tx, parentId int64, name string, pattern string, description string, isBreak bool, condsJSON []byte) (int64, error) {
op := NewHTTPLocationOperator()
op.IsOn = true
@@ -105,7 +106,7 @@ func (this *HTTPLocationDAO) CreateLocation(tx *dbs.Tx, parentId int64, name str
return types.Int64(op.Id), nil
}
// UpdateLocation 修改路规则
// UpdateLocation 修改路规则
func (this *HTTPLocationDAO) UpdateLocation(tx *dbs.Tx, locationId int64, name string, pattern string, description string, isOn bool, isBreak bool, condsJSON []byte) error {
if locationId <= 0 {
return errors.New("invalid locationId")
@@ -130,7 +131,16 @@ func (this *HTTPLocationDAO) UpdateLocation(tx *dbs.Tx, locationId int64, name s
}
// ComposeLocationConfig 组合配置
func (this *HTTPLocationDAO) ComposeLocationConfig(tx *dbs.Tx, locationId int64) (*serverconfigs.HTTPLocationConfig, error) {
func (this *HTTPLocationDAO) ComposeLocationConfig(tx *dbs.Tx, locationId int64, cacheMap maps.Map) (*serverconfigs.HTTPLocationConfig, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(locationId)
var cacheConfig = cacheMap.Get(cacheKey)
if cacheConfig != nil {
return cacheConfig.(*serverconfigs.HTTPLocationConfig), nil
}
location, err := this.FindEnabledHTTPLocation(tx, locationId)
if err != nil {
return nil, err
@@ -150,7 +160,7 @@ func (this *HTTPLocationDAO) ComposeLocationConfig(tx *dbs.Tx, locationId int64)
// web
if location.WebId > 0 {
webConfig, err := SharedHTTPWebDAO.ComposeWebConfig(tx, int64(location.WebId))
webConfig, err := SharedHTTPWebDAO.ComposeWebConfig(tx, int64(location.WebId), cacheMap)
if err != nil {
return nil, err
}
@@ -166,7 +176,7 @@ func (this *HTTPLocationDAO) ComposeLocationConfig(tx *dbs.Tx, locationId int64)
}
config.ReverseProxyRef = ref
if ref.ReverseProxyId > 0 {
reverseProxyConfig, err := SharedReverseProxyDAO.ComposeReverseProxyConfig(tx, ref.ReverseProxyId)
reverseProxyConfig, err := SharedReverseProxyDAO.ComposeReverseProxyConfig(tx, ref.ReverseProxyId, cacheMap)
if err != nil {
return nil, err
}
@@ -184,6 +194,8 @@ func (this *HTTPLocationDAO) ComposeLocationConfig(tx *dbs.Tx, locationId int64)
config.Conds = conds
}
cacheMap[cacheKey] = config
return config, nil
}
@@ -247,13 +259,13 @@ func (this *HTTPLocationDAO) UpdateLocationWeb(tx *dbs.Tx, locationId int64, web
}
// ConvertLocationRefs 转换引用为配置
func (this *HTTPLocationDAO) ConvertLocationRefs(tx *dbs.Tx, refs []*serverconfigs.HTTPLocationRef) (locations []*serverconfigs.HTTPLocationConfig, err error) {
func (this *HTTPLocationDAO) ConvertLocationRefs(tx *dbs.Tx, refs []*serverconfigs.HTTPLocationRef, cacheMap maps.Map) (locations []*serverconfigs.HTTPLocationConfig, err error) {
for _, ref := range refs {
config, err := this.ComposeLocationConfig(tx, ref.LocationId)
config, err := this.ComposeLocationConfig(tx, ref.LocationId, cacheMap)
if err != nil {
return nil, err
}
children, err := this.ConvertLocationRefs(tx, ref.Children)
children, err := this.ConvertLocationRefs(tx, ref.Children, cacheMap)
if err != nil {
return nil, err
}
@@ -275,6 +287,16 @@ func (this *HTTPLocationDAO) FindEnabledLocationIdWithWebId(tx *dbs.Tx, webId in
FindInt64Col(0)
}
// FindEnabledLocationIdWithReverseProxyId 查找包含某个反向代理的路由规则
func (this *HTTPLocationDAO) FindEnabledLocationIdWithReverseProxyId(tx *dbs.Tx, reverseProxyId int64) (serverId int64, err error) {
return this.Query(tx).
State(ServerStateEnabled).
Where("JSON_CONTAINS(reverseProxy, :jsonQuery)").
Param("jsonQuery", maps.Map{"reverseProxyId": reverseProxyId}.AsJSON()).
ResultPk().
FindInt64Col(0)
}
// NotifyUpdate 通知更新
func (this *HTTPLocationDAO) NotifyUpdate(tx *dbs.Tx, locationId int64) error {
webId, err := SharedHTTPWebDAO.FindEnabledWebIdWithLocationId(tx, locationId)

View File

@@ -1,6 +1,6 @@
package models
// 路径规则配置
// HTTPLocation 路由规则配置
type HTTPLocation struct {
Id uint32 `field:"id"` // ID
TemplateId uint32 `field:"templateId"` // 模版ID

View File

@@ -7,6 +7,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
@@ -36,12 +37,12 @@ func init() {
})
}
// 初始化
// Init 初始化
func (this *HTTPPageDAO) Init() {
_ = this.DAOObject.Init()
}
// 启用条目
// EnableHTTPPage 启用条目
func (this *HTTPPageDAO) EnableHTTPPage(tx *dbs.Tx, pageId int64) error {
_, err := this.Query(tx).
Pk(pageId).
@@ -53,7 +54,7 @@ func (this *HTTPPageDAO) EnableHTTPPage(tx *dbs.Tx, pageId int64) error {
return this.NotifyUpdate(tx, pageId)
}
// 禁用条目
// DisableHTTPPage 禁用条目
func (this *HTTPPageDAO) DisableHTTPPage(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -62,7 +63,7 @@ func (this *HTTPPageDAO) DisableHTTPPage(tx *dbs.Tx, id int64) error {
return err
}
// 查找启用中的条目
// FindEnabledHTTPPage 查找启用中的条目
func (this *HTTPPageDAO) FindEnabledHTTPPage(tx *dbs.Tx, id int64) (*HTTPPage, error) {
result, err := this.Query(tx).
Pk(id).
@@ -74,7 +75,7 @@ func (this *HTTPPageDAO) FindEnabledHTTPPage(tx *dbs.Tx, id int64) (*HTTPPage, e
return result.(*HTTPPage), err
}
// 创建Page
// CreatePage 创建Page
func (this *HTTPPageDAO) CreatePage(tx *dbs.Tx, statusList []string, url string, newStatus int) (pageId int64, err error) {
op := NewHTTPPageOperator()
op.IsOn = true
@@ -97,7 +98,7 @@ func (this *HTTPPageDAO) CreatePage(tx *dbs.Tx, statusList []string, url string,
return types.Int64(op.Id), nil
}
// 修改Page
// UpdatePage 修改Page
func (this *HTTPPageDAO) UpdatePage(tx *dbs.Tx, pageId int64, statusList []string, url string, newStatus int) error {
if pageId <= 0 {
return errors.New("invalid pageId")
@@ -126,8 +127,17 @@ func (this *HTTPPageDAO) UpdatePage(tx *dbs.Tx, pageId int64, statusList []strin
return this.NotifyUpdate(tx, pageId)
}
// 组合配置
func (this *HTTPPageDAO) ComposePageConfig(tx *dbs.Tx, pageId int64) (*serverconfigs.HTTPPageConfig, error) {
// ComposePageConfig 组合配置
func (this *HTTPPageDAO) ComposePageConfig(tx *dbs.Tx, pageId int64, cacheMap maps.Map) (*serverconfigs.HTTPPageConfig, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(pageId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*serverconfigs.HTTPPageConfig), nil
}
page, err := this.FindEnabledHTTPPage(tx, pageId)
if err != nil {
return nil, err
@@ -154,10 +164,12 @@ func (this *HTTPPageDAO) ComposePageConfig(tx *dbs.Tx, pageId int64) (*servercon
}
}
cacheMap[cacheKey] = config
return config, nil
}
// 通知更新
// NotifyUpdate 通知更新
func (this *HTTPPageDAO) NotifyUpdate(tx *dbs.Tx, pageId int64) error {
webId, err := SharedHTTPWebDAO.FindEnabledWebIdWithPageId(tx, pageId)
if err != nil {

View File

@@ -8,6 +8,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
@@ -76,7 +77,16 @@ func (this *HTTPRewriteRuleDAO) FindEnabledHTTPRewriteRule(tx *dbs.Tx, id int64)
}
// ComposeRewriteRule 构造配置
func (this *HTTPRewriteRuleDAO) ComposeRewriteRule(tx *dbs.Tx, rewriteRuleId int64) (*serverconfigs.HTTPRewriteRule, error) {
func (this *HTTPRewriteRuleDAO) ComposeRewriteRule(tx *dbs.Tx, rewriteRuleId int64, cacheMap maps.Map) (*serverconfigs.HTTPRewriteRule, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(rewriteRuleId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*serverconfigs.HTTPRewriteRule), nil
}
rule, err := this.FindEnabledHTTPRewriteRule(tx, rewriteRuleId)
if err != nil {
return nil, err
@@ -105,6 +115,9 @@ func (this *HTTPRewriteRuleDAO) ComposeRewriteRule(tx *dbs.Tx, rewriteRuleId int
}
config.Conds = conds
}
cacheMap[cacheKey] = config
return config, nil
}

View File

@@ -75,7 +75,16 @@ func (this *HTTPWebDAO) FindEnabledHTTPWeb(tx *dbs.Tx, id int64) (*HTTPWeb, erro
}
// ComposeWebConfig 组合配置
func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfigs.HTTPWebConfig, error) {
func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64, cacheMap maps.Map) (*serverconfigs.HTTPWebConfig, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":config:" + types.String(webId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*serverconfigs.HTTPWebConfig), nil
}
web, err := SharedHTTPWebDAO.FindEnabledHTTPWeb(tx, webId)
if err != nil {
return nil, err
@@ -181,7 +190,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
return nil, err
}
for index, page := range pages {
pageConfig, err := SharedHTTPPageDAO.ComposePageConfig(tx, page.Id)
pageConfig, err := SharedHTTPPageDAO.ComposePageConfig(tx, page.Id, cacheMap)
if err != nil {
return nil, err
}
@@ -235,7 +244,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
// 自定义防火墙设置
if firewallRef.FirewallPolicyId > 0 {
firewallPolicy, err := SharedHTTPFirewallPolicyDAO.ComposeFirewallPolicy(tx, firewallRef.FirewallPolicyId)
firewallPolicy, err := SharedHTTPFirewallPolicyDAO.ComposeFirewallPolicy(tx, firewallRef.FirewallPolicyId, cacheMap)
if err != nil {
return nil, err
}
@@ -247,7 +256,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
}
}
// 路规则
// 路规则
if IsNotNull(web.Locations) {
refs := []*serverconfigs.HTTPLocationRef{}
err = json.Unmarshal([]byte(web.Locations), &refs)
@@ -257,7 +266,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
if len(refs) > 0 {
config.LocationRefs = refs
locations, err := SharedHTTPLocationDAO.ConvertLocationRefs(tx, refs)
locations, err := SharedHTTPLocationDAO.ConvertLocationRefs(tx, refs, cacheMap)
if err != nil {
return nil, err
}
@@ -302,7 +311,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
return nil, err
}
for _, ref := range refs {
rewriteRule, err := SharedHTTPRewriteRuleDAO.ComposeRewriteRule(tx, ref.RewriteRuleId)
rewriteRule, err := SharedHTTPRewriteRuleDAO.ComposeRewriteRule(tx, ref.RewriteRuleId, cacheMap)
if err != nil {
return nil, err
}
@@ -356,7 +365,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
}
var newRefs []*serverconfigs.HTTPAuthPolicyRef
for _, ref := range authConfig.PolicyRefs {
policyConfig, err := SharedHTTPAuthPolicyDAO.ComposePolicyConfig(tx, ref.AuthPolicyId)
policyConfig, err := SharedHTTPAuthPolicyDAO.ComposePolicyConfig(tx, ref.AuthPolicyId, cacheMap)
if err != nil {
return nil, err
}
@@ -368,6 +377,8 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64) (*serverconfig
config.Auth = authConfig
}
cacheMap[cacheKey] = config
return config, nil
}
@@ -563,7 +574,7 @@ func (this *HTTPWebDAO) UpdateWebFirewall(tx *dbs.Tx, webId int64, firewallJSON
return this.NotifyUpdate(tx, webId)
}
// UpdateWebLocations 更改路规则配置
// UpdateWebLocations 更改路规则配置
func (this *HTTPWebDAO) UpdateWebLocations(tx *dbs.Tx, webId int64, locationsJSON []byte) error {
if webId <= 0 {
return errors.New("invalid webId")

View File

@@ -23,7 +23,7 @@ type HTTPWeb struct {
Gzip string `field:"gzip"` // Gzip配置
Cache string `field:"cache"` // 缓存配置
Firewall string `field:"firewall"` // 防火墙设置
Locations string `field:"locations"` // 路规则配置
Locations string `field:"locations"` // 路规则配置
Websocket string `field:"websocket"` // Websocket设置
RewriteRules string `field:"rewriteRules"` // 重写规则配置
HostRedirects string `field:"hostRedirects"` // 域名跳转
@@ -53,7 +53,7 @@ type HTTPWebOperator struct {
Gzip interface{} // Gzip配置
Cache interface{} // 缓存配置
Firewall interface{} // 防火墙设置
Locations interface{} // 路规则配置
Locations interface{} // 路规则配置
Websocket interface{} // Websocket设置
RewriteRules interface{} // 重写规则配置
HostRedirects interface{} // 域名跳转

View File

@@ -3,6 +3,7 @@ package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/utils"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
@@ -86,6 +87,16 @@ func (this *IPItemDAO) FindEnabledIPItem(tx *dbs.Tx, id int64) (*IPItem, error)
return result.(*IPItem), err
}
// DisableOldIPItem 根据IP删除以前的旧记录
func (this *IPItemDAO) DisableOldIPItem(tx *dbs.Tx, listId int64, ipFrom string, ipTo string) error {
return this.Query(tx).
Attr("listId", listId).
Attr("ipFrom", ipFrom).
Attr("ipTo", ipTo).
Set("state", IPItemStateDisabled).
UpdateQuickly()
}
// CreateIPItem 创建IP
func (this *IPItemDAO) CreateIPItem(tx *dbs.Tx, listId int64, ipFrom string, ipTo string, expiredAt int64, reason string, itemType IPItemType, eventLevel string) (int64, error) {
version, err := SharedIPListDAO.IncreaseVersion(tx)
@@ -229,6 +240,20 @@ func (this *IPItemDAO) FindEnabledItemContainsIP(tx *dbs.Tx, listId int64, ip ui
return one.(*IPItem), nil
}
// FindEnabledItemsWithIP 根据IP查找Item
func (this *IPItemDAO) FindEnabledItemsWithIP(tx *dbs.Tx, ip string) (result []*IPItem, err error) {
_, err = this.Query(tx).
Attr("ipFrom", ip).
Attr("ipTo", "").
Where("(expiredAt=0 OR expiredAt>:nowTime)").
Param("nowTime", time.Now().Unix()).
Where("listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1)").
AscPk().
Slice(&result).
FindAll()
return
}
// ExistsEnabledItem 检查IP是否存在
func (this *IPItemDAO) ExistsEnabledItem(tx *dbs.Tx, itemId int64) (bool, error) {
return this.Query(tx).
@@ -292,7 +317,7 @@ func (this *IPItemDAO) NotifyUpdate(tx *dbs.Tx, itemId int64) error {
if len(resultClusterIds) > 0 {
for _, clusterId := range resultClusterIds {
err = SharedNodeTaskDAO.CreateClusterTask(tx, clusterId, NodeTaskTypeIPItemChanged)
err = SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, NodeTaskTypeIPItemChanged)
if err != nil {
return err
}

View File

@@ -2,6 +2,7 @@ package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ipconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
@@ -253,7 +254,7 @@ func (this *IPListDAO) NotifyUpdate(tx *dbs.Tx, listId int64, taskType NodeTaskT
if len(resultClusterIds) > 0 {
for _, clusterId := range resultClusterIds {
err = SharedNodeTaskDAO.CreateClusterTask(tx, clusterId, taskType)
err = SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, taskType)
if err != nil {
return err
}

View File

@@ -4,6 +4,7 @@ import (
"crypto/md5"
"fmt"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
@@ -28,8 +29,8 @@ const (
MessageTypeHealthCheckFailed MessageType = "HealthCheckFailed" // 节点健康检查失败
MessageTypeHealthCheckNodeUp MessageType = "HealthCheckNodeUp" // 因健康检查节点上线
MessageTypeHealthCheckNodeDown MessageType = "HealthCheckNodeDown" // 因健康检查节点下线
MessageTypeNodeInactive MessageType = "NodeInactive" // 节点不活跃
MessageTypeNodeActive MessageType = "NodeActive" // 节点活跃
MessageTypeNodeInactive MessageType = "NodeInactive" // 边缘节点不活跃
MessageTypeNodeActive MessageType = "NodeActive" // 边缘节点活跃
MessageTypeClusterDNSSyncFailed MessageType = "ClusterDNSSyncFailed" // DNS同步失败
MessageTypeSSLCertExpiring MessageType = "SSLCertExpiring" // SSL证书即将过期
MessageTypeSSLCertACMETaskFailed MessageType = "SSLCertACMETaskFailed" // SSL证书任务执行失败
@@ -38,6 +39,16 @@ const (
MessageTypeServerNamesAuditingSuccess MessageType = "ServerNamesAuditingSuccess" // 服务域名审核成功
MessageTypeServerNamesAuditingFailed MessageType = "ServerNamesAuditingFailed" // 服务域名审核失败
MessageTypeThresholdSatisfied MessageType = "ThresholdSatisfied" // 满足阈值
MessageTypeFirewallEvent MessageType = "FirewallEvent" // 防火墙事件
MessageTypeIPAddrUp MessageType = "IPAddrUp" // IP地址上线
MessageTypeIPAddrDown MessageType = "IPAddrDown" // IP地址下线
MessageTypeNSNodeInactive MessageType = "NSNodeInactive" // NS节点不活跃
MessageTypeNSNodeActive MessageType = "NSNodeActive" // NS节点活跃
MessageTypeReportNodeInactive MessageType = "ReportNodeInactive" // 区域监控节点节点不活跃
MessageTypeReportNodeActive MessageType = "ReportNodeActive" // 区域监控节点活跃
MessageTypeConnectivity MessageType = "Connectivity"
)
type MessageDAO dbs.DAO
@@ -92,18 +103,14 @@ func (this *MessageDAO) FindEnabledMessage(tx *dbs.Tx, id int64) (*Message, erro
}
// CreateClusterMessage 创建集群消息
func (this *MessageDAO) CreateClusterMessage(tx *dbs.Tx, clusterId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
_, err := this.createMessage(tx, clusterId, 0, messageType, level, subject, body, paramsJSON)
func (this *MessageDAO) CreateClusterMessage(tx *dbs.Tx, role string, clusterId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
_, err := this.createMessage(tx, role, clusterId, 0, messageType, level, subject, body, paramsJSON)
if err != nil {
return err
}
// 发送给媒介接收人
err = SharedMessageTaskDAO.CreateMessageTasks(tx, MessageTaskTarget{
ClusterId: clusterId,
NodeId: 0,
ServerId: 0,
}, messageType, subject, body)
err = SharedMessageTaskDAO.CreateMessageTasks(tx, role, 0, 0, 0, messageType, subject, body)
if err != nil {
return err
}
@@ -112,47 +119,33 @@ func (this *MessageDAO) CreateClusterMessage(tx *dbs.Tx, clusterId int64, messag
}
// CreateNodeMessage 创建节点消息
func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, role string, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte, force bool) error {
// 检查N分钟内是否已经发送过
hash := this.calHash(subject, body, paramsJSON)
exists, err := this.Query(tx).
Attr("hash", hash).
Gt("createdAt", time.Now().Unix()-10*60). // 10分钟
Exist()
if err != nil {
return err
}
if exists {
return nil
hash := this.calHash(role, clusterId, nodeId, subject, body, paramsJSON)
if !force {
exists, err := this.Query(tx).
Attr("hash", hash).
Gt("createdAt", time.Now().Unix()-10*60). // 10分钟
Exist()
if err != nil {
return err
}
if exists {
return nil
}
}
_, err = this.createMessage(tx, clusterId, nodeId, messageType, level, subject, body, paramsJSON)
_, err := this.createMessage(tx, role, clusterId, nodeId, messageType, level, subject, body, paramsJSON)
if err != nil {
return err
}
// 发送给媒介接收人 - 集群
err = SharedMessageTaskDAO.CreateMessageTasks(tx, MessageTaskTarget{
ClusterId: clusterId,
NodeId: 0,
ServerId: 0,
}, messageType, subject, body)
err = SharedMessageTaskDAO.CreateMessageTasks(tx, role, clusterId, nodeId, 0, messageType, subject, body)
if err != nil {
return err
}
// 发送给媒介接收人 - 节点
if nodeId > 0 {
err = SharedMessageTaskDAO.CreateMessageTasks(tx, MessageTaskTarget{
ClusterId: clusterId,
NodeId: nodeId,
ServerId: 0,
}, messageType, subject, body)
if err != nil {
return err
}
}
return nil
}
@@ -178,7 +171,7 @@ func (this *MessageDAO) CreateMessage(tx *dbs.Tx, adminId int64, userId int64, m
op.State = MessageStateEnabled
op.IsRead = false
op.Day = timeutil.Format("Ymd")
op.Hash = this.calHash(subject, body, paramsJSON)
op.Hash = this.calHash(nodeconfigs.NodeRoleAdmin, 0, 0, subject, body, paramsJSON)
err := this.Save(tx, op)
if err != nil {
return err
@@ -286,13 +279,14 @@ func (this *MessageDAO) CheckMessageUser(tx *dbs.Tx, messageId int64, adminId in
}
// 创建消息
func (this *MessageDAO) createMessage(tx *dbs.Tx, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) (int64, error) {
func (this *MessageDAO) createMessage(tx *dbs.Tx, role string, clusterId int64, nodeId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) (int64, error) {
// TODO 检查同样的消息最近是否发送过
// 创建新消息
op := NewMessageOperator()
op.AdminId = 0 // TODO
op.UserId = 0 // TODO
op.Role = role
op.ClusterId = clusterId
op.NodeId = nodeId
op.Type = messageType
@@ -313,7 +307,7 @@ func (this *MessageDAO) createMessage(tx *dbs.Tx, clusterId int64, nodeId int64,
op.State = MessageStateEnabled
op.CreatedAt = time.Now().Unix()
op.Day = timeutil.Format("Ymd")
op.Hash = this.calHash(subject, body, paramsJSON)
op.Hash = this.calHash(role, clusterId, nodeId, subject, body, paramsJSON)
err := this.Save(tx, op)
if err != nil {
@@ -323,10 +317,11 @@ func (this *MessageDAO) createMessage(tx *dbs.Tx, clusterId int64, nodeId int64,
}
// 计算Hash
func (this *MessageDAO) calHash(subject string, body string, paramsJSON []byte) string {
func (this *MessageDAO) calHash(role string, clusterId int64, nodeId int64, subject string, body string, paramsJSON []byte) string {
h := md5.New()
h.Write([]byte(subject))
h.Write([]byte(body))
h.Write([]byte(role + "@" + types.String(clusterId) + "@" + types.String(nodeId)))
h.Write([]byte(subject + "@"))
h.Write([]byte(body + "@"))
h.Write(paramsJSON)
return fmt.Sprintf("%x", h.Sum(nil))
}

View File

@@ -1,6 +1,7 @@
package models
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/dbs"
"testing"
@@ -11,7 +12,7 @@ func TestMessageDAO_CreateClusterMessage(t *testing.T) {
var tx *dbs.Tx
dao := NewMessageDAO()
err := dao.CreateClusterMessage(tx, 1, "test", "error", "123", "123", []byte("456"))
err := dao.CreateClusterMessage(tx, nodeconfigs.NodeRoleNode, 1, "test", "error", "123", "123", []byte("456"))
if err != nil {
t.Fatal(err)
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
)
const (
@@ -35,7 +36,7 @@ func init() {
})
}
// 启用条目
// EnableMessageMediaInstance 启用条目
func (this *MessageMediaInstanceDAO) EnableMessageMediaInstance(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -44,7 +45,7 @@ func (this *MessageMediaInstanceDAO) EnableMessageMediaInstance(tx *dbs.Tx, id i
return err
}
// 禁用条目
// DisableMessageMediaInstance 禁用条目
func (this *MessageMediaInstanceDAO) DisableMessageMediaInstance(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
@@ -53,20 +54,32 @@ func (this *MessageMediaInstanceDAO) DisableMessageMediaInstance(tx *dbs.Tx, id
return err
}
// 查找启用中的条目
func (this *MessageMediaInstanceDAO) FindEnabledMessageMediaInstance(tx *dbs.Tx, id int64) (*MessageMediaInstance, error) {
// FindEnabledMessageMediaInstance 查找启用中的条目
func (this *MessageMediaInstanceDAO) FindEnabledMessageMediaInstance(tx *dbs.Tx, instanceId int64, cacheMap maps.Map) (*MessageMediaInstance, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":record:" + types.String(instanceId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*MessageMediaInstance), nil
}
result, err := this.Query(tx).
Pk(id).
Pk(instanceId).
Attr("state", MessageMediaInstanceStateEnabled).
Find()
if result == nil {
return nil, err
}
cacheMap[cacheKey] = result
return result.(*MessageMediaInstance), err
}
// 创建媒介实例
func (this *MessageMediaInstanceDAO) CreateMediaInstance(tx *dbs.Tx, name string, mediaType string, params maps.Map, description string) (int64, error) {
// CreateMediaInstance 创建媒介实例
func (this *MessageMediaInstanceDAO) CreateMediaInstance(tx *dbs.Tx, name string, mediaType string, params maps.Map, description string, rateJSON []byte, hashLifeSeconds int32) (int64, error) {
op := NewMessageMediaInstanceOperator()
op.Name = name
op.MediaType = mediaType
@@ -83,13 +96,18 @@ func (this *MessageMediaInstanceDAO) CreateMediaInstance(tx *dbs.Tx, name string
op.Description = description
if len(rateJSON) > 0 {
op.Rate = rateJSON
}
op.HashLife = hashLifeSeconds
op.IsOn = true
op.State = MessageMediaInstanceStateEnabled
return this.SaveInt64(tx, op)
}
// 修改媒介实例
func (this *MessageMediaInstanceDAO) UpdateMediaInstance(tx *dbs.Tx, instanceId int64, name string, mediaType string, params maps.Map, description string, isOn bool) error {
// UpdateMediaInstance 修改媒介实例
func (this *MessageMediaInstanceDAO) UpdateMediaInstance(tx *dbs.Tx, instanceId int64, name string, mediaType string, params maps.Map, description string, rateJSON []byte, hashLifeSeconds int32, isOn bool) error {
if instanceId <= 0 {
return errors.New("invalid instanceId")
}
@@ -109,12 +127,18 @@ func (this *MessageMediaInstanceDAO) UpdateMediaInstance(tx *dbs.Tx, instanceId
}
op.Params = paramsJSON
if len(rateJSON) > 0 {
op.Rate = rateJSON
}
op.HashLife = hashLifeSeconds
op.Description = description
op.IsOn = isOn
return this.Save(tx, op)
}
// 计算接收人数量
// CountAllEnabledMediaInstances 计算接收人数量
func (this *MessageMediaInstanceDAO) CountAllEnabledMediaInstances(tx *dbs.Tx, mediaType string, keyword string) (int64, error) {
query := this.Query(tx)
if len(mediaType) > 0 {
@@ -130,7 +154,7 @@ func (this *MessageMediaInstanceDAO) CountAllEnabledMediaInstances(tx *dbs.Tx, m
Count()
}
// 列出单页接收人
// ListAllEnabledMediaInstances 列出单页接收人
func (this *MessageMediaInstanceDAO) ListAllEnabledMediaInstances(tx *dbs.Tx, mediaType string, keyword string, offset int64, size int64) (result []*MessageMediaInstance, err error) {
query := this.Query(tx)
if len(mediaType) > 0 {
@@ -150,3 +174,15 @@ func (this *MessageMediaInstanceDAO) ListAllEnabledMediaInstances(tx *dbs.Tx, me
FindAll()
return
}
// FindInstanceHashLifeSeconds 获取单个实例的HashLife
func (this *MessageMediaInstanceDAO) FindInstanceHashLifeSeconds(tx *dbs.Tx, instanceId int64) (int32, error) {
hashLife, err := this.Query(tx).
Pk(instanceId).
Result("hashLife").
FindIntCol(0)
if err != nil {
return 0, err
}
return types.Int32(hashLife), nil
}

View File

@@ -1,6 +1,6 @@
package models
// 消息媒介接收人
// MessageMediaInstance 消息媒介接收人
type MessageMediaInstance struct {
Id uint32 `field:"id"` // ID
Name string `field:"name"` // 名称
@@ -8,7 +8,9 @@ type MessageMediaInstance struct {
MediaType string `field:"mediaType"` // 媒介类型
Params string `field:"params"` // 媒介参数
Description string `field:"description"` // 备注
Rate string `field:"rate"` // 发送频率
State uint8 `field:"state"` // 状态
HashLife int32 `field:"hashLife"` // HASH有效期
}
type MessageMediaInstanceOperator struct {
@@ -18,7 +20,9 @@ type MessageMediaInstanceOperator struct {
MediaType interface{} // 媒介类型
Params interface{} // 媒介参数
Description interface{} // 备注
Rate interface{} // 发送频率
State interface{} // 状态
HashLife interface{} // HASH有效期
}
func NewMessageMediaInstanceOperator() *MessageMediaInstanceOperator {

View File

@@ -5,6 +5,7 @@ type Message struct {
Id uint64 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
UserId uint32 `field:"userId"` // 用户ID
Role string `field:"role"` // 角色
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
Level string `field:"level"` // 级别
@@ -23,6 +24,7 @@ type MessageOperator struct {
Id interface{} // ID
AdminId interface{} // 管理员ID
UserId interface{} // 用户ID
Role interface{} // 角色
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
Level interface{} // 级别

View File

@@ -75,11 +75,12 @@ func (this *MessageReceiverDAO) DisableReceivers(tx *dbs.Tx, clusterId int64, no
}
// CreateReceiver 创建接收人
func (this *MessageReceiverDAO) CreateReceiver(tx *dbs.Tx, target MessageTaskTarget, messageType MessageType, params maps.Map, recipientId int64, recipientGroupId int64) (int64, error) {
func (this *MessageReceiverDAO) CreateReceiver(tx *dbs.Tx, role string, clusterId int64, nodeId int64, serverId int64, messageType MessageType, params maps.Map, recipientId int64, recipientGroupId int64) (int64, error) {
op := NewMessageReceiverOperator()
op.ClusterId = target.ClusterId
op.NodeId = target.NodeId
op.ServerId = target.ServerId
op.Role = role
op.ClusterId = clusterId
op.NodeId = nodeId
op.ServerId = serverId
op.Type = messageType
if params == nil {
@@ -98,15 +99,16 @@ func (this *MessageReceiverDAO) CreateReceiver(tx *dbs.Tx, target MessageTaskTar
}
// FindAllEnabledReceivers 查询接收人
func (this *MessageReceiverDAO) FindAllEnabledReceivers(tx *dbs.Tx, target MessageTaskTarget, messageType string) (result []*MessageReceiver, err error) {
func (this *MessageReceiverDAO) FindAllEnabledReceivers(tx *dbs.Tx, role string, clusterId int64, nodeId int64, serverId int64, messageType string) (result []*MessageReceiver, err error) {
query := this.Query(tx)
if len(messageType) > 0 {
query.Attr("type", []string{"*", messageType}) // *表示所有的
}
_, err = query.
Attr("clusterId", target.ClusterId).
Attr("nodeId", target.NodeId).
Attr("serverId", target.ServerId).
Attr("role", role).
Attr("clusterId", clusterId).
Attr("nodeId", nodeId).
Attr("serverId", serverId).
State(MessageReceiverStateEnabled).
AscPk().
Slice(&result).
@@ -115,15 +117,102 @@ func (this *MessageReceiverDAO) FindAllEnabledReceivers(tx *dbs.Tx, target Messa
}
// CountAllEnabledReceivers 计算接收人数量
func (this *MessageReceiverDAO) CountAllEnabledReceivers(tx *dbs.Tx, target MessageTaskTarget, messageType string) (int64, error) {
func (this *MessageReceiverDAO) CountAllEnabledReceivers(tx *dbs.Tx, role string, clusterId int64, nodeId int64, serverId int64, messageType string) (int64, error) {
query := this.Query(tx)
if len(messageType) > 0 {
query.Attr("type", []string{"*", messageType}) // *表示所有的
}
return query.
Attr("clusterId", target.ClusterId).
Attr("nodeId", target.NodeId).
Attr("serverId", target.ServerId).
Attr("role", role).
Attr("clusterId", clusterId).
Attr("nodeId", nodeId).
Attr("serverId", serverId).
State(MessageReceiverStateEnabled).
Count()
}
// FindEnabledBestFitReceivers 查询最适合的接收人
func (this *MessageReceiverDAO) FindEnabledBestFitReceivers(tx *dbs.Tx, role string, clusterId int64, nodeId int64, serverId int64, messageType string) (result []*MessageReceiver, err error) {
// serverId优先
query := this.Query(tx)
if len(messageType) > 0 {
query.Attr("type", []string{"*", messageType}) // *表示所有的
}
if len(role) > 0 {
query.Attr("role", role)
}
if serverId > 0 {
query.Attr("serverId", serverId)
} else if nodeId > 0 {
query.Attr("nodeId", nodeId)
} else if clusterId > 0 {
query.Attr("clusterId", clusterId)
}
_, err = query.
State(MessageReceiverStateEnabled).
AscPk().
Slice(&result).
FindAll()
if err != nil || len(result) > 0 {
return
}
// nodeId优先
if serverId > 0 && nodeId > 0 {
query = this.Query(tx)
if len(messageType) > 0 {
query.Attr("type", []string{"*", messageType}) // *表示所有的
}
if len(role) > 0 {
query.Attr("role", role)
}
query.Attr("nodeId", nodeId)
_, err = query.
State(MessageReceiverStateEnabled).
AscPk().
Slice(&result).
FindAll()
if err != nil || len(result) > 0 {
return
}
}
// clusterId优先
if (serverId > 0 || nodeId > 0) && clusterId > 0 {
query = this.Query(tx)
if len(messageType) > 0 {
query.Attr("type", []string{"*", messageType}) // *表示所有的
}
if len(role) > 0 {
query.Attr("role", role)
}
query.Attr("clusterId", clusterId)
_, err = query.
State(MessageReceiverStateEnabled).
AscPk().
Slice(&result).
FindAll()
if err != nil || len(result) > 0 {
return
}
}
// 去掉集群ID
query = this.Query(tx)
if len(messageType) > 0 {
query.Attr("type", []string{"*", messageType}) // *表示所有的
}
if len(role) > 0 {
query.Attr("role", role)
}
_, err = query.
State(MessageReceiverStateEnabled).
AscPk().
Slice(&result).
FindAll()
if err != nil || len(result) > 0 {
return
}
return
}

View File

@@ -1,6 +1,30 @@
package models
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/logs"
"testing"
)
func TestMessageReceiverDAO_FindEnabledBestFitReceivers(t *testing.T) {
var tx *dbs.Tx
{
receivers, err := NewMessageReceiverDAO().FindEnabledBestFitReceivers(tx, nodeconfigs.NodeRoleNode, 18, 1, 2, "*")
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(receivers, t)
}
{
receivers, err := NewMessageReceiverDAO().FindEnabledBestFitReceivers(tx, nodeconfigs.NodeRoleNode, 30, 1, 2, "*")
if err != nil {
t.Fatal(err)
}
logs.PrintAsJSON(receivers, t)
}
}

View File

@@ -3,6 +3,7 @@ package models
// MessageReceiver 消息通知接收人
type MessageReceiver struct {
Id uint32 `field:"id"` // ID
Role string `field:"role"` // 节点角色
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
ServerId uint32 `field:"serverId"` // 服务ID
@@ -15,6 +16,7 @@ type MessageReceiver struct {
type MessageReceiverOperator struct {
Id interface{} // ID
Role interface{} // 节点角色
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
ServerId interface{} // 服务ID

View File

@@ -7,6 +7,9 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"regexp"
)
const (
@@ -54,19 +57,32 @@ func (this *MessageRecipientDAO) DisableMessageRecipient(tx *dbs.Tx, id int64) e
}
// FindEnabledMessageRecipient 查找启用中的条目
func (this *MessageRecipientDAO) FindEnabledMessageRecipient(tx *dbs.Tx, id int64) (*MessageRecipient, error) {
func (this *MessageRecipientDAO) FindEnabledMessageRecipient(tx *dbs.Tx, recipientId int64, cacheMap maps.Map,
) (*MessageRecipient, error) {
if cacheMap == nil {
cacheMap = maps.Map{}
}
var cacheKey = this.Table + ":record:" + types.String(recipientId)
var cache = cacheMap.Get(cacheKey)
if cache != nil {
return cache.(*MessageRecipient), nil
}
result, err := this.Query(tx).
Pk(id).
Pk(recipientId).
Attr("state", MessageRecipientStateEnabled).
Find()
if result == nil {
return nil, err
}
cacheMap[cacheKey] = result
return result.(*MessageRecipient), err
}
// CreateRecipient 创建接收人
func (this *MessageRecipientDAO) CreateRecipient(tx *dbs.Tx, adminId int64, instanceId int64, user string, groupIds []int64, description string) (int64, error) {
func (this *MessageRecipientDAO) CreateRecipient(tx *dbs.Tx, adminId int64, instanceId int64, user string, groupIds []int64, description string, timeFrom string, timeTo string) (int64, error) {
op := NewMessageRecipientOperator()
op.AdminId = adminId
op.InstanceId = instanceId
@@ -83,13 +99,22 @@ func (this *MessageRecipientDAO) CreateRecipient(tx *dbs.Tx, adminId int64, inst
}
op.GroupIds = groupIdsJSON
// 判断格式
var timeReg = regexp.MustCompile(`^\d+:\d+:\d+$`)
if timeReg.MatchString(timeFrom) {
op.TimeFrom = timeFrom
}
if timeReg.MatchString(timeTo) {
op.TimeTo = timeTo
}
op.IsOn = true
op.State = MessageRecipientStateEnabled
return this.SaveInt64(tx, op)
}
// UpdateRecipient 修改接收人
func (this *MessageRecipientDAO) UpdateRecipient(tx *dbs.Tx, recipientId int64, adminId int64, instanceId int64, user string, groupIds []int64, description string, isOn bool) error {
func (this *MessageRecipientDAO) UpdateRecipient(tx *dbs.Tx, recipientId int64, adminId int64, instanceId int64, user string, groupIds []int64, description string, timeFrom string, timeTo string, isOn bool) error {
if recipientId <= 0 {
return errors.New("invalid recipientId")
}
@@ -111,6 +136,20 @@ func (this *MessageRecipientDAO) UpdateRecipient(tx *dbs.Tx, recipientId int64,
op.GroupIds = groupIdsJSON
op.Description = description
// 判断格式
var timeReg = regexp.MustCompile(`^\d+:\d+:\d+$`)
if timeReg.MatchString(timeFrom) {
op.TimeFrom = timeFrom
} else {
op.TimeFrom = dbs.SQL("NULL")
}
if timeReg.MatchString(timeTo) {
op.TimeTo = timeTo
} else {
op.TimeTo = dbs.SQL("NULL")
}
op.IsOn = isOn
return this.Save(tx, op)
}
@@ -187,3 +226,11 @@ func (this *MessageRecipientDAO) FindAllEnabledAndOnRecipientIdsWithGroup(tx *db
}
return result, nil
}
// FindRecipientInstanceId 查找接收人的媒介
func (this *MessageRecipientDAO) FindRecipientInstanceId(tx *dbs.Tx, recipientId int64) (int64, error) {
return this.Query(tx).
Pk(recipientId).
Result("instanceId").
FindInt64Col(0)
}

View File

@@ -1,6 +1,6 @@
package models
// 消息媒介接收人
// MessageRecipient 消息媒介接收人
type MessageRecipient struct {
Id uint32 `field:"id"` // ID
AdminId uint32 `field:"adminId"` // 管理员ID
@@ -9,6 +9,8 @@ type MessageRecipient struct {
User string `field:"user"` // 接收人信息
GroupIds string `field:"groupIds"` // 分组ID
State uint8 `field:"state"` // 状态
TimeFrom string `field:"timeFrom"` // 开始时间
TimeTo string `field:"timeTo"` // 结束时间
Description string `field:"description"` // 备注
}
@@ -20,6 +22,8 @@ type MessageRecipientOperator struct {
User interface{} // 接收人信息
GroupIds interface{} // 分组ID
State interface{} // 状态
TimeFrom interface{} // 开始时间
TimeTo interface{} // 结束时间
Description interface{} // 备注
}

View File

@@ -1,9 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package models
type MessageTaskTarget struct {
ClusterId int64
NodeId int64
ServerId int64
}

View File

@@ -2,9 +2,15 @@ package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
@@ -35,6 +41,21 @@ func NewMessageTaskDAO() *MessageTaskDAO {
var SharedMessageTaskDAO *MessageTaskDAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedMessageTaskDAO.CleanExpiredMessageTasks(nil, 30) // 只保留30天
if err != nil {
remotelogs.Error("SharedMessageTaskDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func init() {
dbs.OnReady(func() {
SharedMessageTaskDAO = NewMessageTaskDAO()
@@ -73,13 +94,46 @@ func (this *MessageTaskDAO) FindEnabledMessageTask(tx *dbs.Tx, id int64) (*Messa
// CreateMessageTask 创建任务
func (this *MessageTaskDAO) CreateMessageTask(tx *dbs.Tx, recipientId int64, instanceId int64, user string, subject string, body string, isPrimary bool) (int64, error) {
var hash = stringutil.Md5(types.String(recipientId) + "@" + types.String(instanceId) + "@" + user + "@" + subject + "@" + types.String(isPrimary))
recipientInstanceId, err := SharedMessageRecipientDAO.FindRecipientInstanceId(tx, recipientId)
if err != nil {
return 0, err
}
if recipientInstanceId > 0 {
hashLifeSeconds, err := SharedMessageMediaInstanceDAO.FindInstanceHashLifeSeconds(tx, recipientInstanceId)
if err != nil {
return 0, err
}
if hashLifeSeconds >= 0 { // 意味着此值如果小于0则不做判断
lastMessageAt, err := this.Query(tx).
Attr("hash", hash).
Result("createdAt").
DescPk().
FindInt64Col(0)
if err != nil {
return 0, err
}
// 对于同一个人N分钟内消息不重复发送
if hashLifeSeconds <= 0 {
hashLifeSeconds = 60
}
if lastMessageAt > 0 && time.Now().Unix()-lastMessageAt < int64(hashLifeSeconds) {
return 0, nil
}
}
}
op := NewMessageTaskOperator()
op.RecipientId = recipientId
op.InstanceId = instanceId
op.Hash = hash
op.User = user
op.Subject = subject
op.Body = body
op.IsPrimary = isPrimary
op.Day = timeutil.Format("Ymd")
op.Status = MessageTaskStatusNone
op.State = MessageTaskStateEnabled
return this.SaveInt64(tx, op)
@@ -93,6 +147,8 @@ func (this *MessageTaskDAO) FindSendingMessageTasks(tx *dbs.Tx, size int64) (res
_, err = this.Query(tx).
State(MessageTaskStateEnabled).
Attr("status", MessageTaskStatusNone).
Where("(recipientId=0 OR recipientId IN (SELECT id FROM "+SharedMessageRecipientDAO.Table+" WHERE state=1 AND isOn=1 AND (timeFrom IS NULL OR timeTo IS NULL OR :time BETWEEN timeFrom AND timeTo)))").
Param("time", timeutil.Format("H:i:s")).
Desc("isPrimary").
AscPk().
Limit(size).
@@ -101,6 +157,28 @@ func (this *MessageTaskDAO) FindSendingMessageTasks(tx *dbs.Tx, size int64) (res
return
}
// CountMessageTasksWithStatus 根据状态计算任务数量
func (this *MessageTaskDAO) CountMessageTasksWithStatus(tx *dbs.Tx, status MessageTaskStatus) (int64, error) {
return this.Query(tx).
State(MessageTaskStateEnabled).
Attr("status", status).
Count()
}
// ListMessageTasksWithStatus 根据状态列出单页任务
func (this *MessageTaskDAO) ListMessageTasksWithStatus(tx *dbs.Tx, status MessageTaskStatus, offset int64, size int64) (result []*MessageTask, err error) {
_, err = this.Query(tx).
State(MessageTaskStateEnabled).
Attr("status", status).
Desc("isPrimary").
AscPk().
Offset(offset).
Limit(size).
Slice(&result).
FindAll()
return
}
// UpdateMessageTaskStatus 设置发送的状态
func (this *MessageTaskDAO) UpdateMessageTaskStatus(tx *dbs.Tx, taskId int64, status MessageTaskStatus, result []byte) error {
if taskId <= 0 {
@@ -117,8 +195,8 @@ func (this *MessageTaskDAO) UpdateMessageTaskStatus(tx *dbs.Tx, taskId int64, st
}
// CreateMessageTasks 从集群、节点或者服务中创建任务
func (this *MessageTaskDAO) CreateMessageTasks(tx *dbs.Tx, target MessageTaskTarget, messageType MessageType, subject string, body string) error {
receivers, err := SharedMessageReceiverDAO.FindAllEnabledReceivers(tx, target, messageType)
func (this *MessageTaskDAO) CreateMessageTasks(tx *dbs.Tx, role nodeconfigs.NodeRole, clusterId int64, nodeId int64, serverId int64, messageType MessageType, subject string, body string) error {
receivers, err := SharedMessageReceiverDAO.FindEnabledBestFitReceivers(tx, role, clusterId, nodeId, serverId, messageType)
if err != nil {
return err
}
@@ -150,3 +228,16 @@ func (this *MessageTaskDAO) CreateMessageTasks(tx *dbs.Tx, target MessageTaskTar
return nil
}
// CleanExpiredMessageTasks 清理
func (this *MessageTaskDAO) CleanExpiredMessageTasks(tx *dbs.Tx, days int) error {
if days <= 0 {
days = 30
}
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Where("(day IS NULL OR day<:day)").
Param("day", day).
Delete()
return err
}

View File

@@ -3,4 +3,20 @@ package models
import (
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestMessageTaskDAO_FindSendingMessageTasks(t *testing.T) {
dbs.NotifyReady()
var tx *dbs.Tx
tasks, err := NewMessageTaskDAO().FindSendingMessageTasks(tx, 100)
if err != nil {
t.Fatal(err)
}
t.Log(len(tasks), "tasks")
for _, task := range tasks {
t.Log("task:", task.Id, "recipient:", task.RecipientId)
}
}

View File

@@ -1,13 +1,32 @@
package models
import (
"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"time"
)
type MessageTaskLogDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedMessageTaskLogDAO.CleanExpiredLogs(nil, 30) // 只保留30天
if err != nil {
remotelogs.Error("SharedMessageTaskLogDAO", "clean expired data failed: "+err.Error())
}
}
}()
})
}
func NewMessageTaskLogDAO() *MessageTaskLogDAO {
return dbs.NewDAO(&MessageTaskLogDAO{
DAOObject: dbs.DAOObject{
@@ -27,25 +46,28 @@ func init() {
})
}
// 创建日志
// CreateLog 创建日志
func (this *MessageTaskLogDAO) CreateLog(tx *dbs.Tx, taskId int64, isOk bool, errMsg string, response string) error {
op := NewMessageTaskLogOperator()
op.TaskId = taskId
op.IsOk = isOk
op.Error = errMsg
op.Response = response
op.Day = timeutil.Format("Ymd")
return this.Save(tx, op)
}
// 计算日志数量
// CountLogs 计算日志数量
func (this *MessageTaskLogDAO) CountLogs(tx *dbs.Tx) (int64, error) {
return this.Query(tx).
Where("taskId IN (SELECT id FROM " + SharedMessageTaskDAO.Table + ")").
Count()
}
// 列出单页日志
// ListLogs 列出单页日志
func (this *MessageTaskLogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64) (result []*MessageTaskLog, err error) {
_, err = this.Query(tx).
Where("taskId IN (SELECT id FROM " + SharedMessageTaskDAO.Table + ")").
Offset(offset).
Limit(size).
DescPk().
@@ -53,3 +75,16 @@ func (this *MessageTaskLogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64) (r
FindAll()
return
}
// CleanExpiredLogs 清理
func (this *MessageTaskLogDAO) CleanExpiredLogs(tx *dbs.Tx, days int) error {
if days <= 0 {
days = 30
}
var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
_, err := this.Query(tx).
Where("(day IS NULL OR day<:day)").
Param("day", day).
Delete()
return err
}

View File

@@ -1,6 +1,6 @@
package models
// 消息发送日志
// MessageTaskLog 消息发送日志
type MessageTaskLog struct {
Id uint64 `field:"id"` // ID
TaskId uint64 `field:"taskId"` // 任务ID
@@ -8,6 +8,7 @@ type MessageTaskLog struct {
IsOk uint8 `field:"isOk"` // 是否成功
Error string `field:"error"` // 错误信息
Response string `field:"response"` // 响应信息
Day string `field:"day"` // YYYYMMDD
}
type MessageTaskLogOperator struct {
@@ -17,6 +18,7 @@ type MessageTaskLogOperator struct {
IsOk interface{} // 是否成功
Error interface{} // 错误信息
Response interface{} // 响应信息
Day interface{} // YYYYMMDD
}
func NewMessageTaskLogOperator() *MessageTaskLogOperator {

View File

@@ -1,9 +1,10 @@
package models
//
// MessageTask 消息发送相关任务
type MessageTask struct {
Id uint64 `field:"id"` // ID
RecipientId uint32 `field:"recipientId"` // 接收人ID
Hash string `field:"hash"` // SUM标识
InstanceId uint32 `field:"instanceId"` // 媒介实例ID
User string `field:"user"` // 接收用户标识
Subject string `field:"subject"` // 标题
@@ -13,12 +14,14 @@ type MessageTask struct {
SentAt uint64 `field:"sentAt"` // 最后一次发送时间
State uint8 `field:"state"` // 状态
Result string `field:"result"` // 结果
Day string `field:"day"` // YYYYMMDD
IsPrimary uint8 `field:"isPrimary"` // 是否优先
}
type MessageTaskOperator struct {
Id interface{} // ID
RecipientId interface{} // 接收人ID
Hash interface{} // SUM标识
InstanceId interface{} // 媒介实例ID
User interface{} // 接收用户标识
Subject interface{} // 标题
@@ -28,6 +31,7 @@ type MessageTaskOperator struct {
SentAt interface{} // 最后一次发送时间
State interface{} // 状态
Result interface{} // 结果
Day interface{} // YYYYMMDD
IsPrimary interface{} // 是否优先
}

View File

@@ -0,0 +1,177 @@
package models
import (
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
)
const (
MetricChartStateEnabled = 1 // 已启用
MetricChartStateDisabled = 0 // 已禁用
)
type MetricChartDAO dbs.DAO
func NewMetricChartDAO() *MetricChartDAO {
return dbs.NewDAO(&MetricChartDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeMetricCharts",
Model: new(MetricChart),
PkName: "id",
},
}).(*MetricChartDAO)
}
var SharedMetricChartDAO *MetricChartDAO
func init() {
dbs.OnReady(func() {
SharedMetricChartDAO = NewMetricChartDAO()
})
}
// EnableMetricChart 启用条目
func (this *MetricChartDAO) EnableMetricChart(tx *dbs.Tx, chartId int64) error {
_, err := this.Query(tx).
Pk(chartId).
Set("state", MetricChartStateEnabled).
Update()
return err
}
// DisableMetricChart 禁用条目
func (this *MetricChartDAO) DisableMetricChart(tx *dbs.Tx, chartId int64) error {
_, err := this.Query(tx).
Pk(chartId).
Set("state", MetricChartStateDisabled).
Update()
return err
}
// FindEnabledMetricChart 查找启用中的条目
func (this *MetricChartDAO) FindEnabledMetricChart(tx *dbs.Tx, chartId int64) (*MetricChart, error) {
result, err := this.Query(tx).
Pk(chartId).
Attr("state", MetricChartStateEnabled).
Find()
if result == nil {
return nil, err
}
return result.(*MetricChart), err
}
// FindMetricChartName 根据主键查找名称
func (this *MetricChartDAO) FindMetricChartName(tx *dbs.Tx, chartId int64) (string, error) {
return this.Query(tx).
Pk(chartId).
Result("name").
FindStringCol("")
}
// CreateChart 创建图表
func (this *MetricChartDAO) CreateChart(tx *dbs.Tx, itemId int64, name string, chartType string, widthDiv int32, maxItems int32, params maps.Map, ignoreEmptyKeys bool, ignoredKeys []string) (int64, error) {
op := NewMetricChartOperator()
op.ItemId = itemId
op.Name = name
op.Type = chartType
op.WidthDiv = widthDiv
op.MaxItems = maxItems
if params == nil {
params = maps.Map{}
}
paramsJSON, err := json.Marshal(params)
if err != nil {
return 0, err
}
op.Params = paramsJSON
op.IgnoreEmptyKeys = ignoreEmptyKeys
if len(ignoredKeys) == 0 {
op.IgnoredKeys = "[]"
} else {
ignoredKeysJSON, err := json.Marshal(ignoredKeys)
if err != nil {
return 0, err
}
op.IgnoredKeys = ignoredKeysJSON
}
op.IsOn = true
op.State = MetricChartStateEnabled
return this.SaveInt64(tx, op)
}
// UpdateChart 修改图表
func (this *MetricChartDAO) UpdateChart(tx *dbs.Tx, chartId int64, name string, chartType string, widthDiv int32, maxItems int32, params maps.Map, ignoreEmptyKeys bool, ignoredKeys []string, isOn bool) error {
if chartId <= 0 {
return errors.New("invalid chartId")
}
op := NewMetricChartOperator()
op.Id = chartId
op.Name = name
op.Type = chartType
op.WidthDiv = widthDiv
op.MaxItems = maxItems
if params == nil {
params = maps.Map{}
}
paramsJSON, err := json.Marshal(params)
if err != nil {
return err
}
op.Params = paramsJSON
op.IgnoreEmptyKeys = ignoreEmptyKeys
if len(ignoredKeys) == 0 {
op.IgnoredKeys = "[]"
} else {
ignoredKeysJSON, err := json.Marshal(ignoredKeys)
if err != nil {
return err
}
op.IgnoredKeys = ignoredKeysJSON
}
op.IsOn = isOn
return this.Save(tx, op)
}
// CountEnabledCharts 计算图表数量
func (this *MetricChartDAO) CountEnabledCharts(tx *dbs.Tx, itemId int64) (int64, error) {
return this.Query(tx).
Attr("itemId", itemId).
State(MetricChartStateEnabled).
Count()
}
// ListEnabledCharts 列出单页图表
func (this *MetricChartDAO) ListEnabledCharts(tx *dbs.Tx, itemId int64, offset int64, size int64) (result []*MetricChart, err error) {
_, err = this.Query(tx).
Attr("itemId", itemId).
State(MetricChartStateEnabled).
Offset(offset).
Limit(size).
DescPk().
Slice(&result).
FindAll()
return
}
// FindAllEnabledCharts 查找所有图表
func (this *MetricChartDAO) FindAllEnabledCharts(tx *dbs.Tx, itemId int64) (result []*MetricChart, err error) {
_, err = this.Query(tx).
Attr("itemId", itemId).
State(MetricChartStateEnabled).
DescPk().
Slice(&result).
FindAll()
return
}

View File

@@ -1,4 +1,4 @@
package metrics
package models
import (
_ "github.com/go-sql-driver/mysql"

View File

@@ -0,0 +1,38 @@
package models
// MetricChart 指标图表
type MetricChart struct {
Id uint32 `field:"id"` // ID
ItemId uint32 `field:"itemId"` // 指标ID
Name string `field:"name"` // 名称
Code string `field:"code"` // 代号
Type string `field:"type"` // 图形类型
WidthDiv int32 `field:"widthDiv"` // 宽度划分
Params string `field:"params"` // 图形参数
Order uint32 `field:"order"` // 排序
IsOn uint8 `field:"isOn"` // 是否启用
State uint8 `field:"state"` // 状态
MaxItems uint32 `field:"maxItems"` // 最多条目
IgnoreEmptyKeys uint8 `field:"ignoreEmptyKeys"` // 忽略空的键值
IgnoredKeys string `field:"ignoredKeys"` // 忽略键值
}
type MetricChartOperator struct {
Id interface{} // ID
ItemId interface{} // 指标ID
Name interface{} // 名称
Code interface{} // 代号
Type interface{} // 图形类型
WidthDiv interface{} // 宽度划分
Params interface{} // 图形参数
Order interface{} // 排序
IsOn interface{} // 是否启用
State interface{} // 状态
MaxItems interface{} // 最多条目
IgnoreEmptyKeys interface{} // 忽略空的键值
IgnoredKeys interface{} // 忽略键值
}
func NewMetricChartOperator() *MetricChartOperator {
return &MetricChartOperator{}
}

View File

@@ -0,0 +1,17 @@
package models
import "encoding/json"
func (this *MetricChart) DecodeIgnoredKeys() []string {
if len(this.IgnoredKeys) == 0 {
return []string{}
}
var result = []string{}
err := json.Unmarshal([]byte(this.IgnoredKeys), &result)
if err != nil {
// 这里忽略错误
return result
}
return result
}

View File

@@ -1,13 +1,16 @@
package metrics
package models
import (
"encoding/json"
"github.com/TeaOSLab/EdgeAPI/internal/errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/types"
"sort"
"strings"
)
const (
@@ -46,12 +49,35 @@ func (this *MetricItemDAO) EnableMetricItem(tx *dbs.Tx, id int64) error {
}
// DisableMetricItem 禁用条目
func (this *MetricItemDAO) DisableMetricItem(tx *dbs.Tx, id int64) error {
_, err := this.Query(tx).
Pk(id).
func (this *MetricItemDAO) DisableMetricItem(tx *dbs.Tx, itemId int64) error {
isPublic, err := this.Query(tx).
Pk(itemId).
Result("isPublic").
FindIntCol(0)
if err != nil {
return err
}
_, err = this.Query(tx).
Pk(itemId).
Set("state", MetricItemStateDisabled).
Update()
return err
if err != nil {
return err
}
// 通知更新
err = this.NotifyUpdate(tx, itemId, isPublic == 1)
if err != nil {
return err
}
// 删除统计数据
err = SharedMetricStatDAO.DeleteItemStats(tx, itemId)
if err != nil {
return err
}
return nil
}
// FindEnabledMetricItem 查找启用中的条目
@@ -75,7 +101,9 @@ func (this *MetricItemDAO) FindMetricItemName(tx *dbs.Tx, id int64) (string, err
}
// CreateItem 创建指标
func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string, name string, keys []string, period int32, periodUnit string, value string) (int64, error) {
func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string, name string, keys []string, period int32, periodUnit string, value string, isPublic bool) (int64, error) {
sort.Strings(keys)
op := NewMetricItemOperator()
op.Code = code
op.Category = category
@@ -92,16 +120,47 @@ func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string,
op.Period = period
op.PeriodUnit = periodUnit
op.Value = value
op.IsPublic = isPublic
op.IsOn = true
op.State = MetricItemStateEnabled
return this.SaveInt64(tx, op)
itemId, err := this.SaveInt64(tx, op)
if err != nil {
return 0, err
}
if isPublic {
err = this.NotifyUpdate(tx, itemId, isPublic)
if err != nil {
return 0, err
}
}
return itemId, nil
}
// UpdateItem 修改\指标
func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, keys []string, period int32, periodUnit string, value string, isOn bool) error {
func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, keys []string, period int32, periodUnit string, value string, isOn bool, isPublic bool) error {
if itemId <= 0 {
return errors.New("invalid itemId")
}
sort.Strings(keys)
// 是否有变化
oldItem, err := this.FindEnabledMetricItem(tx, itemId)
if err != nil {
return err
}
if oldItem == nil {
return nil
}
oldIsPublic := oldItem.IsPublic == 1
var versionChanged = false
if strings.Join(oldItem.DecodeKeys(), "&") != strings.Join(keys, "&") || types.Int32(oldItem.Period) != period || oldItem.PeriodUnit != periodUnit || oldItem.Value != value {
versionChanged = true
}
// 保存
op := NewMetricItemOperator()
op.Id = itemId
op.Name = name
@@ -118,7 +177,32 @@ func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, key
op.PeriodUnit = periodUnit
op.Value = value
op.IsOn = isOn
return this.Save(tx, op)
if versionChanged {
op.Version = dbs.SQL("version+1")
}
op.IsPublic = isPublic
err = this.Save(tx, op)
if err != nil {
return err
}
// 通知更新
if versionChanged || (oldItem.IsOn == 0 && isOn) || (oldItem.IsOn == 1 && !isOn) || oldIsPublic != isPublic {
err := this.NotifyUpdate(tx, itemId, isPublic || oldIsPublic)
if err != nil {
return err
}
}
// 删除旧数据
if versionChanged {
err := SharedMetricStatDAO.DeleteOldVersionItemStats(tx, itemId, types.Int32(oldItem.Version+1))
if err != nil {
return err
}
}
return nil
}
// CountEnabledItems 计算指标的数量
@@ -144,6 +228,18 @@ func (this *MetricItemDAO) ListEnabledItems(tx *dbs.Tx, category serverconfigs.M
return
}
// FindAllPublicItems 取得公用的指标
func (this *MetricItemDAO) FindAllPublicItems(tx *dbs.Tx) (result []*MetricItem, err error) {
_, err = this.Query(tx).
State(MetricItemStateEnabled).
Attr("userId", 0).
Attr("isPublic", true).
DescPk().
Slice(&result).
FindAll()
return
}
// ComposeItemConfig 组合指标配置
func (this *MetricItemDAO) ComposeItemConfig(tx *dbs.Tx, itemId int64) (*serverconfigs.MetricItemConfig, error) {
if itemId <= 0 {
@@ -168,7 +264,67 @@ func (this *MetricItemDAO) ComposeItemConfig(tx *dbs.Tx, itemId int64) (*serverc
Category: item.Category,
Value: item.Value,
Keys: item.DecodeKeys(),
Version: types.Int32(item.Version),
}
return config, nil
}
// ComposeItemConfigWithItem 根据Item信息组合指标
func (this *MetricItemDAO) ComposeItemConfigWithItem(item *MetricItem) *serverconfigs.MetricItemConfig {
if item == nil {
return nil
}
var config = &serverconfigs.MetricItemConfig{
Id: int64(item.Id),
IsOn: item.IsOn == 1,
Period: types.Int(item.Period),
PeriodUnit: item.PeriodUnit,
Category: item.Category,
Value: item.Value,
Keys: item.DecodeKeys(),
Version: types.Int32(item.Version),
}
return config
}
// FindItemVersion 获取指标的版本号
func (this *MetricItemDAO) FindItemVersion(tx *dbs.Tx, itemId int64) (int32, error) {
version, err := this.Query(tx).
Pk(itemId).
Result("version").
FindIntCol(0)
if err != nil {
return 0, err
}
return types.Int32(version), nil
}
// NotifyUpdate 通知更新
func (this *MetricItemDAO) NotifyUpdate(tx *dbs.Tx, itemId int64, isPublic bool) error {
if isPublic {
clusterIds, err := SharedNodeClusterDAO.FindAllEnableClusterIds(tx)
if err != nil {
return err
}
for _, clusterId := range clusterIds {
err = SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, NodeTaskTypeConfigChanged)
if err != nil {
return err
}
}
return nil
}
clusterIds, err := SharedNodeClusterMetricItemDAO.FindAllClusterIdsWithItemId(tx, itemId)
if err != nil {
return err
}
for _, clusterId := range clusterIds {
err = SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, NodeTaskTypeConfigChanged)
if err != nil {
return err
}
}
return nil
}

View File

@@ -1,4 +1,4 @@
package metrics
package models
import (
_ "github.com/go-sql-driver/mysql"

View File

@@ -1,4 +1,4 @@
package metrics
package models
// MetricItem 指标定义
type MetricItem struct {
@@ -14,6 +14,8 @@ type MetricItem struct {
PeriodUnit string `field:"periodUnit"` // 周期单位
Value string `field:"value"` // 值运算
State uint8 `field:"state"` // 状态
Version uint32 `field:"version"` // 版本号
IsPublic uint8 `field:"isPublic"` // 是否为公用
}
type MetricItemOperator struct {
@@ -29,6 +31,8 @@ type MetricItemOperator struct {
PeriodUnit interface{} // 周期单位
Value interface{} // 值运算
State interface{} // 状态
Version interface{} // 版本号
IsPublic interface{} // 是否为公用
}
func NewMetricItemOperator() *MetricItemOperator {

View File

@@ -1,7 +1,10 @@
package metrics
package models
import "encoding/json"
import (
"encoding/json"
)
// DecodeKeys 解析Key
func (this *MetricItem) DecodeKeys() []string {
var result []string
if len(this.Keys) > 0 {

View File

@@ -0,0 +1,472 @@
package models
import (
"encoding/json"
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/rands"
timeutil "github.com/iwind/TeaGo/utils/time"
"strconv"
"time"
)
type MetricStatDAO dbs.DAO
func init() {
dbs.OnReadyDone(func() {
// 清理数据任务
var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
go func() {
for range ticker.C {
err := SharedMetricStatDAO.Clean(nil, 120) // 只保留120天
if err != nil {
logs.Println("SharedMetricStatDAO: clean expired data failed: " + err.Error())
}
}
}()
})
}
func NewMetricStatDAO() *MetricStatDAO {
return dbs.NewDAO(&MetricStatDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeMetricStats",
Model: new(MetricStat),
PkName: "id",
},
}).(*MetricStatDAO)
}
var SharedMetricStatDAO *MetricStatDAO
func init() {
dbs.OnReady(func() {
SharedMetricStatDAO = NewMetricStatDAO()
})
}
// CreateStat 创建统计数据
func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64, nodeId int64, serverId int64, itemId int64, keys []string, value float64, time string, version int32) error {
hash += "@" + strconv.FormatInt(nodeId, 10)
var keysString string
if len(keys) > 0 {
keysJSON, err := json.Marshal(keys)
if err != nil {
return err
}
keysString = string(keysJSON)
} else {
keysString = "[]"
}
return this.Query(tx).
Param("value", value).
InsertOrUpdateQuickly(maps.Map{
"hash": hash,
"clusterId": clusterId,
"nodeId": nodeId,
"serverId": serverId,
"itemId": itemId,
"value": value,
"time": time,
"version": version,
"keys": keysString,
"createdDay": timeutil.Format("Ymd"),
}, maps.Map{
"value": value,
})
}
// DeleteOldVersionItemStats 删除以前版本的统计数据
func (this *MetricStatDAO) DeleteOldVersionItemStats(tx *dbs.Tx, itemId int64, version int32) error {
_, err := this.Query(tx).
Attr("itemId", itemId).
Where("version<:version").
Param("version", version).
Delete()
return err
}
// DeleteItemStats 删除某个指标相关的统计数据
func (this *MetricStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error {
_, err := this.Query(tx).
Attr("itemId", itemId).
Delete()
return err
}
// DeleteNodeItemStats 删除某个节点的统计数据
func (this *MetricStatDAO) DeleteNodeItemStats(tx *dbs.Tx, nodeId int64, serverId int64, itemId int64, time string) error {
_, err := this.Query(tx).
Attr("nodeId", nodeId).
Attr("serverId", serverId).
Attr("itemId", itemId).
Attr("time", time).
Delete()
return err
}
// CountItemStats 计算统计数据数量
func (this *MetricStatDAO) CountItemStats(tx *dbs.Tx, itemId int64, version int32) (int64, error) {
return this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
Count()
}
// ListItemStats 列出单页统计数据
func (this *MetricStatDAO) ListItemStats(tx *dbs.Tx, itemId int64, version int32, offset int64, size int64) (result []*MetricStat, err error) {
_, err = this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
Offset(offset).
Limit(size).
Desc("time").
Desc("serverId").
Desc("value").
Slice(&result).
FindAll()
return
}
// FindItemStatsAtLastTime 取得所有集群最近一次计时前 N 个数据
// 适合每条数据中包含不同的Key的场景
func (this *MetricStatDAO) FindItemStatsAtLastTime(tx *dbs.Tx, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
// 最近一次时间
statOne, err := this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
DescPk().
Find()
if err != nil {
return nil, err
}
if statOne == nil {
return nil, nil
}
var lastStat = statOne.(*MetricStat)
var lastTime = lastStat.Time
var query = this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
Attr("time", lastTime).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("MIN(time) AS time", "SUM(value) AS value", "keys").
Desc("value").
Group("keys").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
return
}
// FindItemStatsWithClusterIdAndLastTime 取得集群最近一次计时前 N 个数据
// 适合每条数据中包含不同的Key的场景
func (this *MetricStatDAO) FindItemStatsWithClusterIdAndLastTime(tx *dbs.Tx, clusterId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
// 最近一次时间
statOne, err := this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
DescPk().
Find()
if err != nil {
return nil, err
}
if statOne == nil {
return nil, nil
}
var lastStat = statOne.(*MetricStat)
var lastTime = lastStat.Time
var query = this.Query(tx).
UseIndex("cluster_item_time").
Attr("clusterId", clusterId).
Attr("itemId", itemId).
Attr("version", version).
Attr("time", lastTime).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("MIN(time) AS time", "SUM(value) AS value", "keys").
Desc("value").
Group("keys").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
return
}
// FindItemStatsWithNodeIdAndLastTime 取得节点最近一次计时前 N 个数据
// 适合每条数据中包含不同的Key的场景
func (this *MetricStatDAO) FindItemStatsWithNodeIdAndLastTime(tx *dbs.Tx, nodeId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
// 最近一次时间
statOne, err := this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
DescPk().
Find()
if err != nil {
return nil, err
}
if statOne == nil {
return nil, nil
}
var lastStat = statOne.(*MetricStat)
var lastTime = lastStat.Time
var query = this.Query(tx).
UseIndex("node_item_time").
Attr("nodeId", nodeId).
Attr("itemId", itemId).
Attr("version", version).
Attr("time", lastTime).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("MIN(time) AS time", "SUM(value) AS value", "keys").
Desc("value").
Group("keys").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
return
}
// FindItemStatsWithServerIdAndLastTime 取得节点最近一次计时前 N 个数据
// 适合每条数据中包含不同的Key的场景
func (this *MetricStatDAO) FindItemStatsWithServerIdAndLastTime(tx *dbs.Tx, serverId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
// 最近一次时间
statOne, err := this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
DescPk().
Find()
if err != nil {
return nil, err
}
if statOne == nil {
return nil, nil
}
var lastStat = statOne.(*MetricStat)
var lastTime = lastStat.Time
var query = this.Query(tx).
UseIndex("server_item_time").
Attr("serverId", serverId).
Attr("itemId", itemId).
Attr("version", version).
Attr("time", lastTime).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("MIN(time) AS time", "SUM(value) AS value", "keys").
Desc("value").
Group("keys").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
return
}
// FindLatestItemStats 取得所有集群上最近 N 个时间的数据
// 适合同个Key在不同时间段的变化场景
func (this *MetricStatDAO) FindLatestItemStats(tx *dbs.Tx, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
var query = this.Query(tx).
Attr("itemId", itemId).
Attr("version", version).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
Desc("time").
Group("time").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
if err != nil {
return nil, err
}
lists.Reverse(result)
return
}
// FindLatestItemStatsWithClusterId 取得集群最近 N 个时间的数据
// 适合同个Key在不同时间段的变化场景
func (this *MetricStatDAO) FindLatestItemStatsWithClusterId(tx *dbs.Tx, clusterId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
var query = this.Query(tx).
Attr("clusterId", clusterId).
Attr("itemId", itemId).
Attr("version", version).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
Desc("time").
Group("time").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
if err != nil {
return nil, err
}
lists.Reverse(result)
return
}
// FindLatestItemStatsWithNodeId 取得节点最近 N 个时间的数据
// 适合同个Key在不同时间段的变化场景
func (this *MetricStatDAO) FindLatestItemStatsWithNodeId(tx *dbs.Tx, nodeId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
var query = this.Query(tx).
Attr("nodeId", nodeId).
Attr("itemId", itemId).
Attr("version", version).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
Desc("time").
Group("time").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
if err != nil {
return nil, err
}
lists.Reverse(result)
return
}
// FindLatestItemStatsWithServerId 取得服务最近 N 个时间的数据
// 适合同个Key在不同时间段的变化场景
func (this *MetricStatDAO) FindLatestItemStatsWithServerId(tx *dbs.Tx, serverId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
var query = this.Query(tx).
Attr("serverId", serverId).
Attr("itemId", itemId).
Attr("version", version).
// TODO 增加更多聚合算法,比如 AVG、MEDIAN、MIN、MAX 等
// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
Desc("time").
Group("time").
Limit(size).
Slice(&result)
if ignoreEmptyKeys {
query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
}
if len(ignoreKeys) > 0 {
ignoreKeysJSON, err := json.Marshal(ignoreKeys)
if err != nil {
return nil, err
}
query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
query.Param("ignoredKeys", string(ignoreKeysJSON))
}
_, err = query.
FindAll()
if err != nil {
return nil, err
}
lists.Reverse(result)
return
}
// Clean 清理数据
func (this *MetricStatDAO) Clean(tx *dbs.Tx, days int64) error {
_, err := this.Query(tx).
Lt("createdDay", timeutil.FormatTime("Ymd", time.Now().Unix()-days*86400)).
Delete()
if err != nil {
return err
}
return nil
}

View File

@@ -0,0 +1,23 @@
package models
import (
_ "github.com/go-sql-driver/mysql"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"testing"
)
func TestNewMetricStatDAO_InsertMany(t *testing.T) {
for i := 0; i <= 10_000_000; i++ {
err := NewMetricStatDAO().CreateStat(nil, types.String(i)+"_v1", 18, int64(rands.Int(0, 10000)), int64(rands.Int(0, 10000)), int64(rands.Int(0, 100)), []string{"/html" + types.String(i)}, 1, timeutil.Format("Ymd"), 0)
if err != nil {
t.Fatal(err)
}
if i % 10000 == 0 {
t.Log(i)
}
}
t.Log("done")
}

View File

@@ -0,0 +1,34 @@
package models
// MetricStat 指标统计数据
type MetricStat struct {
Id uint64 `field:"id"` // ID
Hash string `field:"hash"` // Hash值
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
ServerId uint32 `field:"serverId"` // 服务ID
ItemId uint64 `field:"itemId"` // 指标
Keys string `field:"keys"` // 键值
Value float64 `field:"value"` // 数值
Time string `field:"time"` // 分钟值YYYYMMDDHHII
Version uint32 `field:"version"` // 版本号
CreatedDay string `field:"createdDay"` // YYYYMMDD
}
type MetricStatOperator struct {
Id interface{} // ID
Hash interface{} // Hash值
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
ServerId interface{} // 服务ID
ItemId interface{} // 指标
Keys interface{} // 键值
Value interface{} // 数值
Time interface{} // 分钟值YYYYMMDDHHII
Version interface{} // 版本号
CreatedDay interface{} // YYYYMMDD
}
func NewMetricStatOperator() *MetricStatOperator {
return &MetricStatOperator{}
}

View File

@@ -0,0 +1,12 @@
package models
import "encoding/json"
// DecodeKeys 解析Key
func (this *MetricStat) DecodeKeys() []string {
var result []string
if len(this.Keys) > 0 {
_ = json.Unmarshal([]byte(this.Keys), &result)
}
return result
}

View File

@@ -0,0 +1,136 @@
package models
import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
"github.com/iwind/TeaGo/maps"
)
type MetricSumStatDAO dbs.DAO
func NewMetricSumStatDAO() *MetricSumStatDAO {
return dbs.NewDAO(&MetricSumStatDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeMetricSumStats",
Model: new(MetricSumStat),
PkName: "id",
},
}).(*MetricSumStatDAO)
}
var SharedMetricSumStatDAO *MetricSumStatDAO
func init() {
dbs.OnReady(func() {
SharedMetricSumStatDAO = NewMetricSumStatDAO()
})
}
// UpdateSum 更新统计数据
func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, time string, itemId int64, version int32, count int64, total float32) error {
return this.Query(tx).
InsertOrUpdateQuickly(maps.Map{
"clusterId": clusterId,
"nodeId": nodeId,
"serverId": serverId,
"itemId": itemId,
"version": version,
"time": time,
"count": count,
"total": total,
}, maps.Map{
"count": count,
"total": total,
})
}
// FindNodeServerSum 查找某个服务在某个节点上的统计数据
func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
one, err := this.Query(tx).
Attr("nodeId", nodeId).
Attr("serverId", serverId).
Attr("time", time).
Attr("itemId", itemId).
Attr("version", version).
Find()
if err != nil {
return 0, 0, err
}
if one == nil {
return
}
return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
}
// FindSumAtTime 查找某个时间的统计数据
func (this *MetricSumStatDAO) FindSumAtTime(tx *dbs.Tx, time string, itemId int64, version int32) (count int64, total float32, err error) {
one, err := this.Query(tx).
Attr("time", time).
Attr("itemId", itemId).
Attr("version", version).
Result("SUM(count) AS `count`, SUM(total) AS total").
Find()
if err != nil {
return 0, 0, err
}
if one == nil {
return
}
return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
}
// FindServerSum 查找某个服务的统计数据
func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
one, err := this.Query(tx).
Attr("serverId", serverId).
Attr("time", time).
Attr("itemId", itemId).
Attr("version", version).
Result("SUM(count) AS `count`, SUM(total) AS total").
Find()
if err != nil {
return 0, 0, err
}
if one == nil {
return
}
return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
}
// FindClusterSum 查找集群上的统计数据
func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
one, err := this.Query(tx).
Attr("clusterId", clusterId).
Attr("time", time).
Attr("itemId", itemId).
Attr("version", version).
Result("SUM(count) AS `count`, SUM(total) AS total").
Find()
if err != nil {
return 0, 0, err
}
if one == nil {
return
}
return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
}
// FindNodeSum 查找节点上的统计数据
func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
one, err := this.Query(tx).
Attr("nodeId", nodeId).
Attr("time", time).
Attr("itemId", itemId).
Attr("version", version).
Result("SUM(count) AS `count`, SUM(total) AS total").
Find()
if err != nil {
return 0, 0, err
}
if one == nil {
return
}
return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
}

View File

@@ -1,4 +1,4 @@
package metrics
package models
import (
_ "github.com/go-sql-driver/mysql"

View File

@@ -0,0 +1,30 @@
package models
// MetricSumStat 指标统计总和数据
type MetricSumStat struct {
Id uint64 `field:"id"` // ID
ClusterId uint32 `field:"clusterId"` // 集群ID
NodeId uint32 `field:"nodeId"` // 节点ID
ServerId uint32 `field:"serverId"` // 服务ID
ItemId uint64 `field:"itemId"` // 指标
Count uint64 `field:"count"` // 数量
Total float64 `field:"total"` // 总和
Time string `field:"time"` // 分钟值YYYYMMDDHHII
Version uint32 `field:"version"` // 版本号
}
type MetricSumStatOperator struct {
Id interface{} // ID
ClusterId interface{} // 集群ID
NodeId interface{} // 节点ID
ServerId interface{} // 服务ID
ItemId interface{} // 指标
Count interface{} // 数量
Total interface{} // 总和
Time interface{} // 分钟值YYYYMMDDHHII
Version interface{} // 版本号
}
func NewMetricSumStatOperator() *MetricSumStatOperator {
return &MetricSumStatOperator{}
}

View File

@@ -0,0 +1 @@
package models

View File

@@ -1,28 +0,0 @@
package metrics
import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
)
type MetricKeyDAO dbs.DAO
func NewMetricKeyDAO() *MetricKeyDAO {
return dbs.NewDAO(&MetricKeyDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeMetricKeys",
Model: new(MetricKey),
PkName: "id",
},
}).(*MetricKeyDAO)
}
var SharedMetricKeyDAO *MetricKeyDAO
func init() {
dbs.OnReady(func() {
SharedMetricKeyDAO = NewMetricKeyDAO()
})
}

View File

@@ -1,20 +0,0 @@
package metrics
// MetricKey 指标键值
type MetricKey struct {
Id uint64 `field:"id"` // ID
ItemId uint64 `field:"itemId"` // 指标ID
Value string `field:"value"` // 值
Hash string `field:"hash"` // 对值进行Hash
}
type MetricKeyOperator struct {
Id interface{} // ID
ItemId interface{} // 指标ID
Value interface{} // 值
Hash interface{} // 对值进行Hash
}
func NewMetricKeyOperator() *MetricKeyOperator {
return &MetricKeyOperator{}
}

View File

@@ -1 +0,0 @@
package metrics

View File

@@ -1,28 +0,0 @@
package metrics
import (
_ "github.com/go-sql-driver/mysql"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/dbs"
)
type MetricStatDAO dbs.DAO
func NewMetricStatDAO() *MetricStatDAO {
return dbs.NewDAO(&MetricStatDAO{
DAOObject: dbs.DAOObject{
DB: Tea.Env,
Table: "edgeMetricStats",
Model: new(MetricStat),
PkName: "id",
},
}).(*MetricStatDAO)
}
var SharedMetricStatDAO *MetricStatDAO
func init() {
dbs.OnReady(func() {
SharedMetricStatDAO = NewMetricStatDAO()
})
}

Some files were not shown because too many files have changed in this diff Show More