Compare commits

...

272 Commits

Author SHA1 Message Date
刘祥超
644ada1da9 优化代码 2023-03-12 20:32:15 +08:00
刘祥超
0c40250849 优化代码 2023-03-12 16:36:59 +08:00
刘祥超
1d1134a86d 优化代码 2023-03-12 16:18:16 +08:00
刘祥超
28e7664eb7 修复源站重试时可能产生的file is writing错误 2023-03-12 16:09:06 +08:00
刘祥超
50f3ad641c 优化统计相关程序 2023-03-12 12:19:25 +08:00
刘祥超
cc7cf5f8c5 限制统计系统和浏览器的最大长度,减少随机UserAgent攻击影响 2023-03-11 11:58:05 +08:00
刘祥超
339f0f6e94 上传统计数据时增加单次上传数量限制 2023-03-11 11:21:03 +08:00
刘祥超
f558e43342 根据系统内存调整访问日志队列长度 2023-03-10 22:31:40 +08:00
刘祥超
e374e5c90c 优化命令识别 2023-03-10 22:05:40 +08:00
刘祥超
563b775e49 优化命令执行速度 2023-03-10 22:01:39 +08:00
刘祥超
de9e1a4515 执行一般命令时不运行init()中内容 2023-03-10 15:14:14 +08:00
刘祥超
f64b36f17a WAF cc防护增加“检查请求来源指纹”选项 2023-03-10 10:41:16 +08:00
刘祥超
f0e8c82d31 增加CC防护(开源用户需要自己实现) 2023-03-09 15:15:22 +08:00
刘祥超
5770d43230 WAF cc2尝试使用指纹统计方法 2023-03-08 16:59:44 +08:00
刘祥超
d4944c236f 修复几处测试用例 2023-03-08 10:13:41 +08:00
刘祥超
33c761a187 修复本地数据库无法异步提交事务的Bug 2023-03-07 16:48:03 +08:00
刘祥超
d7e6da8d2c 对本地数据库文件进行加锁 2023-03-07 16:22:32 +08:00
刘祥超
44d1a2415c 集群服务设置增加“记录找不到网站日志”选项 2023-03-07 10:30:55 +08:00
刘祥超
c98ff50f06 暂时取消GET302和POST307的迟滞 2023-03-07 08:51:39 +08:00
刘祥超
8835fcb09e GET302/POST307增加访问迟滞 2023-03-06 16:28:54 +08:00
刘祥超
77c56e58c0 GET302/POST307兼容safari浏览器 2023-03-06 16:27:06 +08:00
刘祥超
72c65ca4ee 修复GET302和POST307关闭连接后无法响应的问题 2023-03-06 16:10:58 +08:00
刘祥超
ab019b0bdc 修复在连接读写优化模式下fastcgi无法正常工作的Bug 2023-03-06 10:33:54 +08:00
刘祥超
9709e45ad2 修改软内存限制设置错误 2023-03-05 21:19:10 +08:00
刘祥超
be1f80003c 增加软内存最大值限制 2023-03-05 12:34:46 +08:00
刘祥超
252fcca383 增加测试用例 2023-03-03 14:28:58 +08:00
刘祥超
04ae8fa4a0 系统内存不足时,尝试自动回收内存 2023-03-02 10:54:25 +08:00
刘祥超
c95bd7776a WAF拦截动作可以设置最大封禁时间,从而实现封禁时间随机 2023-03-01 19:00:08 +08:00
刘祥超
8219167d05 WAF支持忽略全局WAF规则 2023-03-01 16:46:43 +08:00
刘祥超
e0a6881343 上传带宽数据时同时上传流量数据 2023-02-27 10:48:16 +08:00
刘祥超
6e985d7f06 修复GET302和POST307无限循环的问题 2023-02-24 17:02:59 +08:00
刘祥超
66719b05dd 修复WAF验证码不能输入超出6位数字的Bug 2023-02-16 14:44:56 +08:00
刘祥超
7197583fea 增加变量${requestPathLowerExtension} 2023-02-10 10:43:30 +08:00
刘祥超
ce29024eef 写入Agent IP时,不提示id重复错误 2023-02-01 10:18:08 +08:00
刘祥超
e1ac67f7fa 版本更改为0.6.4 2023-02-01 10:07:30 +08:00
刘祥超
01812144dd 优化带宽统计 2023-01-12 19:09:57 +08:00
刘祥超
1c34e49629 优化代码 2023-01-12 19:02:38 +08:00
刘祥超
f233fbfb25 版本号修改为0.6.3 2023-01-11 15:44:53 +08:00
刘祥超
5387115e4a 优化代码 2023-01-11 15:24:48 +08:00
刘祥超
d82c03db23 修复在HTTPS下无法连接Websocket的问题 2023-01-10 21:20:27 +08:00
刘祥超
230c5c3766 版本号修改为0.6.2 2023-01-10 21:18:53 +08:00
刘祥超
927425149e 优化代码 2023-01-10 09:47:56 +08:00
刘祥超
5ce1aab92c 修复域名跳转时没有携带参数的Bug 2023-01-09 20:06:09 +08:00
刘祥超
195742bb26 修复读超时时间(ReadDeadline)导致WAFGET302、POST307延时关闭连接的问题 2023-01-09 15:56:59 +08:00
刘祥超
006cc2912d 版本修改为0.6.1 2023-01-09 15:49:16 +08:00
刘祥超
2d4ba90c3b 改进在自动读超时模式下的Websocket连接 2023-01-09 12:36:33 +08:00
刘祥超
a2e6aaaa18 WAF增加“在IP列表内”操作符/优化部分操作符代号 2023-01-08 10:15:46 +08:00
刘祥超
8e68da7725 集群服务设置增加自动读超时选项 2023-01-07 20:04:05 +08:00
刘祥超
7abb84c880 优化网络连接关闭速度 2023-01-07 10:03:32 +08:00
刘祥超
a17878f5b2 WAF增加包含任一字符串、包含所有字符串操作符 2023-01-06 20:07:15 +08:00
刘祥超
8a8881ac47 IP范围支持多行 2023-01-06 19:14:09 +08:00
刘祥超
c567404b7a 优化连接相关代码 2023-01-05 11:13:35 +08:00
刘祥超
b220b0f48e 优化读取HTTP请求Header和握手超时时间 2023-01-05 00:40:49 +08:00
刘祥超
9609c90d75 边缘节点增加数据读超时,以改进客户端上传数据过慢的问题 2023-01-04 20:43:10 +08:00
刘祥超
2c3c32af5b 优化代码 2023-01-02 10:44:10 +08:00
刘祥超
b4a4b2e9b1 集群服务设置中增加性能设置 2023-01-01 19:27:38 +08:00
刘祥超
c42ff1e1e9 实现UA名单功能 2022-12-30 20:49:43 +08:00
刘祥超
9fed1141c2 默认情况下内容压缩不支持Partial Content 2022-12-30 11:44:07 +08:00
刘祥超
e87f031293 增加CORS自适应跨域 2022-12-29 17:16:42 +08:00
刘祥超
c4bac7f43c 优化代码 2022-12-27 18:58:29 +08:00
刘祥超
47818f972e 自动转换访问域名中的大写字母 2022-12-25 15:23:56 +08:00
刘祥超
218a0300c5 修复测试用例 2022-12-23 18:53:49 +08:00
刘祥超
63f6c4177f 修复测试用例 2022-12-23 18:17:32 +08:00
刘祥超
1830c22a31 增加自动Agent识别 2022-12-22 11:38:59 +08:00
刘祥超
18611e8a7c 写数据超时时断开同客户端连接 2022-12-21 16:11:55 +08:00
刘祥超
c45f7adf04 优化连接相关代码 2022-12-21 15:59:07 +08:00
刘祥超
1a200918a8 不支持CONNECT方法 2022-12-19 16:27:58 +08:00
刘祥超
b942bb776e 国家/地区封禁、省份封禁时支持IP变量 2022-12-18 16:04:12 +08:00
刘祥超
5cf84efccd 优化内容为空的缓存 2022-12-14 15:26:18 +08:00
刘祥超
ebb6ebd10c 修复WAF中反斜杠符号(\)有可能解析错误的Bug 2022-12-14 12:27:07 +08:00
刘祥超
42d0d63cf4 优化代码 2022-12-13 18:08:50 +08:00
刘祥超
96f8f7e925 增加edge-node ip.close IP命令 2022-12-12 19:23:58 +08:00
刘祥超
e7e7214d58 调整慢连接超时算法 2022-12-12 10:04:36 +08:00
刘祥超
ade979a725 向客户端写入数据超时时立即关闭连接 2022-12-10 19:51:05 +08:00
刘祥超
60a8de13e7 TCP单次向客户端写入数据时超过30秒即认为超时 2022-12-10 18:22:00 +08:00
刘祥超
9fa24bed0a 修复WAF记录IP动作时无法不超时的Bug 2022-12-06 11:01:34 +08:00
刘祥超
87bc1a7e03 优化OpenFileCache 2022-12-05 11:16:04 +08:00
刘祥超
1a05f56149 优化缓存相关代码 2022-12-05 10:46:44 +08:00
刘祥超
f88db576e1 优化代码 2022-12-05 09:57:01 +08:00
刘祥超
dc3f26ea1a 减少WAF预读尺寸 2022-12-02 21:08:03 +08:00
刘祥超
6fc30144f7 在edge-node conns命令中显示连接时长 2022-12-02 17:03:16 +08:00
刘祥超
25b0b98bd4 增加默认的源站连接数 2022-12-02 10:39:07 +08:00
刘祥超
27b5817d5e 优化请求限制逻辑,连接关闭时自动终止内容发送 2022-11-29 19:14:46 +08:00
刘祥超
dcb61dfd33 版本号更改为0.6.0 2022-11-29 15:42:21 +08:00
刘祥超
bbcfdbbf5e 优化代码 2022-11-29 15:33:12 +08:00
刘祥超
b2a1bef08f 修复服务WAF配置无法更新的Bug 2022-11-28 18:13:08 +08:00
刘祥超
2b18b5c2ca 修改版本号为0.5.9 2022-11-28 18:08:19 +08:00
刘祥超
6ff030dbd8 编译时加入configs/cluster.template.yaml文件 2022-11-27 14:52:48 +08:00
刘祥超
0ddeef6986 支持使用域名中含有通配符清除缓存数据 2022-11-26 11:05:46 +08:00
刘祥超
976bd3600b 优化OpenFileCache功能 2022-11-25 14:52:04 +08:00
刘祥超
a64047a934 优化配置重载程序 2022-11-25 10:50:57 +08:00
刘祥超
e82f207935 统计API调用时低于一半的采样率返回总统计 2022-11-23 20:23:46 +08:00
刘祥超
61b5316a1f 优化代码 2022-11-23 20:13:34 +08:00
刘祥超
82329aa8b0 修复一处编译错误 2022-11-22 18:40:03 +08:00
刘祥超
7dabd9c19c 在监控系统运行时上报API连接状况 2022-11-22 11:23:39 +08:00
刘祥超
9437acd18c 优化代码 2022-11-21 21:08:47 +08:00
刘祥超
9da7a34edf 节点可以单独设置所使用的API节点地址 2022-11-21 19:55:28 +08:00
刘祥超
b6a5491dcc 优化Partial Content兼容性 2022-11-20 18:07:46 +08:00
刘祥超
bcee658567 优化Partial Content配置编码速度 2022-11-19 23:11:05 +08:00
刘祥超
afc8f7b703 优化Partial Content缓存 2022-11-19 21:20:53 +08:00
刘祥超
7a4b89d2fb 缓存Header中忽略Set-Cookie 2022-11-19 17:35:23 +08:00
刘祥超
c6299a2fb0 减少文件缓存写入次数 2022-11-19 17:23:45 +08:00
刘祥超
8b5d74af9b 进一步提升文件缓存写入速度 2022-11-19 15:55:05 +08:00
刘祥超
a194360a56 Update go.mod 2022-11-18 17:39:49 +08:00
刘祥超
b12f7f69ba 优化代码 2022-11-17 20:28:55 +08:00
刘祥超
06ec4d3fba 优化代码 2022-11-17 10:38:20 +08:00
刘祥超
c209ab912f 优化代码 2022-11-17 10:35:43 +08:00
刘祥超
32720d772d 优化代码 2022-11-17 10:32:26 +08:00
刘祥超
a89c02fd10 请求变量增加${cname},WAF checkpoint增加cname和isCNAME 2022-11-16 15:01:10 +08:00
刘祥超
37ef86b92f 接收HTTP请求时去除域名后面的点符号 2022-11-16 11:25:11 +08:00
刘祥超
4c19c37f49 写入缓存时减少对缓存目录的检查频率 2022-11-15 22:25:49 +08:00
刘祥超
1bb818b5b0 边缘节点支持设置多个缓存目录 2022-11-15 20:42:25 +08:00
刘祥超
825e46458f 优化代码 2022-11-15 10:06:57 +08:00
刘祥超
a42737bd28 缩短节点运行日志队列长度 2022-11-14 16:42:50 +08:00
刘祥超
5f76be2cfd 优化代码 2022-11-13 10:32:12 +08:00
刘祥超
dbddf8a91a 优化代码 2022-11-08 21:37:20 +08:00
刘祥超
6c457f41f6 优化代码 2022-11-08 20:58:17 +08:00
刘祥超
e4b2a650f0 优化代码 2022-11-08 20:19:51 +08:00
刘祥超
913ba95801 优化缓存相关代码 2022-11-08 11:03:37 +08:00
刘祥超
a9f8e39703 修复节点设置的“缓存磁盘容量”不起作用的问题 2022-11-07 21:32:20 +08:00
刘祥超
534f013f59 使用版本号来读取节点任务,提升任务同步稳定性 2022-11-06 12:07:26 +08:00
刘祥超
258380f75c 修复无法回报任务执行失败的问题 2022-11-05 14:56:57 +08:00
刘祥超
8c0e51ec46 域名跳转增加是否忽略端口选项 2022-11-05 14:30:29 +08:00
刘祥超
4c37c7ab84 时钟同步增加是否检查chrony选项 2022-11-03 14:59:26 +08:00
刘祥超
f005da1d5f 防盗链提示增加缓存时间,以提升性能 2022-11-02 15:24:30 +08:00
刘祥超
e99acc4694 版本修改为0.5.8 2022-11-02 15:11:55 +08:00
刘祥超
408357dfcf 优化DDoS防护相关错误提示信息 2022-11-01 17:37:40 +08:00
刘祥超
0109a27c06 上传访问日志发生网络错误时不提交 2022-11-01 14:55:06 +08:00
刘祥超
e6e2dccc42 版本号修改为0.5.7 2022-10-31 19:14:03 +08:00
刘祥超
09dcf0d712 集群全局服务配置中增加多个访问日志相关选项 2022-10-26 17:51:16 +08:00
刘祥超
60aebd9306 URL跳转中增加域名跳转、端口跳转 2022-10-26 16:14:37 +08:00
刘祥超
04191d04d3 节点设置中增加“通过IP名单”选项 2022-10-26 10:42:16 +08:00
刘祥超
b80a5c525f 节点缓存目录所在磁盘空间不足时(<5G),暂停缓存写入,同时启动LFU清理 2022-10-25 15:14:28 +08:00
刘祥超
265c1e5312 WAF参数定义增加优先级,可以让“轻”任务优先执行 2022-10-24 17:57:07 +08:00
刘祥超
2723f705b6 修复在iptables中加入ipv6的错误 2022-10-24 16:37:54 +08:00
刘祥超
b4cddd6341 集群服务设置--访问日志中可以设置是否只记录通用Header 2022-10-24 14:39:18 +08:00
刘祥超
5636a81d48 防盗链功能增加禁止的来源域名 2022-10-24 10:21:23 +08:00
刘祥超
d8059960de 文件缓存索引表取消UNIQUE索引,尽可能避免 sqlite malformed 错误 2022-10-23 20:45:41 +08:00
刘祥超
17af4064af 带宽和流量提交失败时,将在一定时间内重试 2022-10-23 19:41:21 +08:00
刘祥超
15f37d2c93 优化用户服务整体启用和禁用 2022-10-23 16:21:11 +08:00
刘祥超
6dc3aa8cb7 单请求写入时间从1个小时增加到2个小时 2022-10-23 09:52:50 +08:00
刘祥超
900cccf2f1 修复源站Websocket源站读取失败导致的异常错误 2022-10-18 19:43:53 +08:00
刘祥超
1fec88dfc6 优化代码 2022-10-14 15:00:05 +08:00
刘祥超
7da9363336 上传带宽信息时附带区域ID信息 2022-10-11 18:57:35 +08:00
刘祥超
d82e633bba 时钟同步程序每天只提示一次警告信息 2022-10-11 11:31:00 +08:00
刘祥超
b363bbaafd 版本修改为0.5.6 2022-10-01 08:50:12 +08:00
刘祥超
92a20e3c9a 修复Websocket无法正常交互的问题 2022-09-30 16:34:21 +08:00
刘祥超
5742dfb263 修复Websocket响应可能被缓存的问题 2022-09-30 14:55:42 +08:00
刘祥超
0ae63511d5 版本调整为v0.5.5 2022-09-28 18:57:27 +08:00
刘祥超
aa60092c20 修复开启WAF后,自动记录请求Body的Bug 2022-09-28 16:46:05 +08:00
刘祥超
54fc265d24 systemd服务增加BEGIN INIT INFO 2022-09-28 08:17:25 +08:00
刘祥超
a5ac900784 版本修改为0.5.4 2022-09-27 08:05:20 +08:00
刘祥超
4053f1da32 程序延时100ms退出 2022-09-26 16:27:51 +08:00
刘祥超
0374ccd8a8 版本号改为0.5.3.2 2022-09-26 16:27:28 +08:00
刘祥超
1d46c446cf 程序退出时关闭sqlite数据库 2022-09-26 16:14:24 +08:00
刘祥超
54b66805f9 将版本修改为0.5.4 2022-09-26 15:17:06 +08:00
刘祥超
f7afcbde92 版本修改为0.5.3.1 2022-09-26 13:02:46 +08:00
刘祥超
8bec1cf68e 优化时钟同步相关代码 2022-09-25 14:26:46 +08:00
刘祥超
2cd1bb7f95 时钟同步错误提示改成警告 2022-09-25 09:40:28 +08:00
刘祥超
19e6329a2b 部分提示支持繁体中文 2022-09-24 20:02:29 +08:00
刘祥超
fce2879567 Websocket支持自定义响应Header 2022-09-23 14:21:53 +08:00
刘祥超
0973765919 增加防盗链拦截时默认提示文字 2022-09-22 17:52:57 +08:00
刘祥超
827679721e 增加防盗链功能 2022-09-22 16:33:53 +08:00
刘祥超
735279bc7a 删除的IP名单不再写入到本地数据库 2022-09-21 17:06:25 +08:00
刘祥超
3eb2ed9897 IP名单数据库增加定时清理 2022-09-21 16:49:48 +08:00
刘祥超
3a913d98c7 优化服务相关错误和警告日志 2022-09-20 14:58:55 +08:00
刘祥超
9bfcd79e36 缩小日志文件尺寸 2022-09-18 16:42:11 +08:00
刘祥超
a81d610302 优化代码 2022-09-18 16:18:31 +08:00
刘祥超
64b1753c4d 自动尝试安装nftables 2022-09-17 21:08:56 +08:00
刘祥超
afcb5c2957 集群设置中增加服务设置/可以设置不记录服务错误日志 2022-09-16 18:41:47 +08:00
刘祥超
7d0b9208a3 访问日志中增加源站状态码 2022-09-16 10:07:40 +08:00
刘祥超
c0f0ec43bb Websocket也支持失败自动重试 2022-09-16 09:37:49 +08:00
刘祥超
30bd66958c 优化NTP时钟自动同步 2022-09-15 15:59:29 +08:00
刘祥超
fafac1a038 优化系统服务管理 2022-09-15 15:01:19 +08:00
刘祥超
f979f9503e 优化服务安装相关代码 2022-09-15 11:45:29 +08:00
刘祥超
b233c3cc7a 优化命令执行相关代码 2022-09-15 11:14:33 +08:00
刘祥超
597ac936f7 检查synflood时忽略IP白名单和局域网连接 2022-09-14 18:52:26 +08:00
刘祥超
a590254eb3 修复有多个网络出口时,可能无法正确转发UDP消息的问题 2022-09-14 17:18:00 +08:00
刘祥超
0498bcf30c 调整GRPC参数 2022-09-12 21:59:52 +08:00
刘祥超
59f9b5c724 优化netdns设置 2022-09-12 17:43:48 +08:00
刘祥超
80729935b6 优化代码 2022-09-11 19:03:53 +08:00
刘祥超
4ca57fb99c 重启时保留-shm,-wal文件 2022-09-09 09:34:00 +08:00
刘祥超
9b35902ad4 访问日志因尺寸过大无法提交到API节点时,自动去除requestBody后再次尝试 2022-09-07 17:34:57 +08:00
刘祥超
3b8bd09190 优化代码 2022-09-07 14:54:36 +08:00
刘祥超
71a5bc0652 可以使用EdgeRecover环境变量指示恢复数据库 2022-09-07 14:44:36 +08:00
刘祥超
ac6a8c4e85 启动时尝试自动修复损坏的缓存索引数据库 2022-09-07 13:55:36 +08:00
刘祥超
f58a808c3a 改进缓存LFU算法 2022-09-07 11:34:26 +08:00
刘祥超
51037be772 将版本修改为0.5.3 2022-09-05 11:04:46 +08:00
刘祥超
443ff9aff7 改进Captcha认证计数 2022-09-05 10:59:02 +08:00
刘祥超
57cb00edf0 修复无法添加IP到本地防火墙的问题 2022-09-04 17:06:19 +08:00
刘祥超
3fb39b479a 优化IP名单锁 2022-09-03 21:10:13 +08:00
刘祥超
4a1daff143 增加简化的缓存条件设置 2022-09-03 18:16:35 +08:00
刘祥超
dd1dbd424e 修复WAF IP名单测试用例的一个可能异常 2022-09-03 09:56:36 +08:00
刘祥超
305cb4b46e 优化WAF中IP名单 2022-09-03 09:54:25 +08:00
刘祥超
ef90dce29b 优化节点进程退出逻辑 2022-09-02 16:12:58 +08:00
刘祥超
3cb69f4c71 将IP加入黑名单时,同时也会关闭此IP相关的连接 2022-09-02 15:20:58 +08:00
刘祥超
af4cd05df2 增加一处注释 2022-09-01 09:06:21 +08:00
刘祥超
64e0ae80b7 DDoS防护增加秒级连接速率限制 2022-08-31 10:00:38 +08:00
刘祥超
8bba228745 优化代码 2022-08-30 18:49:21 +08:00
刘祥超
8cc06e6707 鉴权成功时也在访问日志中记录鉴权类型 2022-08-30 17:28:58 +08:00
刘祥超
52fdee2eeb 修复访问控制后缓存Key包含认证参数的问题 2022-08-30 12:04:01 +08:00
刘祥超
b5f52dd136 优化鉴权 2022-08-30 11:24:23 +08:00
刘祥超
abda886de5 如果系统安装了ntpdate,则自动尝试利用ntpdate同步时间 2022-08-27 15:28:53 +08:00
刘祥超
18f08525b9 WAF和其他请求关闭连接时更加快速 2022-08-27 10:49:16 +08:00
刘祥超
b2a9a31fe5 增加edge-node bandwidth命令查看实时带宽 2022-08-26 16:47:46 +08:00
刘祥超
f578114aeb 修复HTTPS连接无法记录带宽的问题,优化带宽计算方法 2022-08-26 16:47:42 +08:00
刘祥超
bf4f47fc35 DDoS防护增加单IP TCP新连接速率黑名单 2022-08-26 11:31:43 +08:00
刘祥超
0be951742a WAF优化captcha和js_cookie的失败计数器/增强js_cookie的安全性 2022-08-25 17:06:52 +08:00
刘祥超
59d3d6ae4b WAF标签动作匹配之后可以继续尝试匹配别的分组中的规则集 2022-08-25 16:44:44 +08:00
刘祥超
d061876f7e 增加Javascript Cookie验证 2022-08-25 15:35:32 +08:00
刘祥超
ddaec82415 优化RPC获取服务实例方式 2022-08-24 20:04:46 +08:00
刘祥超
8afd00f00d 删除HTTP/2自动关闭Body的逻辑 2022-08-24 16:43:32 +08:00
刘祥超
0b306f0a22 忽略部分HTTP/2错误提示 2022-08-24 16:30:38 +08:00
刘祥超
b36a36172b 优化代码 2022-08-23 14:53:39 +08:00
刘祥超
770278bbbc 实现IP库检查更新 2022-08-23 14:32:39 +08:00
刘祥超
ca24818571 优化缓存索引内存使用 2022-08-22 09:44:09 +08:00
刘祥超
cd0af22655 优化代码 2022-08-22 08:31:39 +08:00
刘祥超
96cb8d8af7 IP库改为手动初始化 2022-08-21 23:09:47 +08:00
刘祥超
91face15bf 使用新版IP库 2022-08-21 20:37:49 +08:00
刘祥超
6f52df63a5 只有系统内存超过3GB的才缓存Hash 2022-08-20 12:09:15 +08:00
刘祥超
509d81dc66 修复检测是否为高速硬盘只能检查sda/vda设备的问题 2022-08-20 11:59:49 +08:00
刘祥超
817f2a6f91 大幅提升缓存索引性能 2022-08-20 11:47:57 +08:00
刘祥超
d74e10c7a8 缓存任务忽略连接API错误 2022-08-20 08:15:16 +08:00
刘祥超
1a1280b76e 优化代码 2022-08-19 14:50:26 +08:00
刘祥超
85d3b1169f 修复Gif中透明图转换WebP失败的问题 2022-08-19 13:39:46 +08:00
刘祥超
1b32292e0c WebP压缩Gif时,自动跳过转换失败的帧,并只提示最后一次错误 2022-08-19 13:27:18 +08:00
刘祥超
e6ba44fb2f [SQLITE]使用事务批量提交一些缓存相关任务 2022-08-17 21:04:00 +08:00
刘祥超
9b6e022f46 将版本修改为0.5.2 2022-08-17 18:56:48 +08:00
刘祥超
4e9ab19fc0 修复因为fsnotify包引用而无法编译的问题 2022-08-17 12:12:40 +08:00
刘祥超
4d634d8fa5 修复一处HTTPS错误提示 2022-08-15 18:26:23 +08:00
刘祥超
a538282e4f 版本修改为0.5.1 2022-08-15 18:26:10 +08:00
刘祥超
df31921954 优化代码 2022-08-14 16:28:40 +08:00
刘祥超
299abb7b04 缓存本地数据库发生错误时同时提示数据库文件名 2022-08-14 15:17:07 +08:00
刘祥超
85ce63b4d3 优化API命名/优化一处测试用例 2022-08-11 12:44:27 +08:00
刘祥超
9d9ae288bd 优化代码 2022-08-10 14:37:27 +08:00
刘祥超
c0a35eb5e7 优化代码 2022-08-09 22:59:37 +08:00
刘祥超
d1d0ff062b 增加Ln节点向源站的最大连接数 2022-08-08 08:12:49 +08:00
刘祥超
396e8a22c4 改进一处错误提示 2022-08-07 19:02:06 +08:00
刘祥超
4f040db1ef 版本号改为0.5.0 2022-08-07 19:01:45 +08:00
刘祥超
e3cf111344 缓存条件增加If-None-Match和If-Modified-Since是否回源选项;默认不回源,防止源站返回304 2022-08-07 16:37:29 +08:00
刘祥超
28cb3c383d 修复一处注释 2022-08-07 11:18:16 +08:00
刘祥超
10665c0f37 优化代码 2022-08-07 11:12:29 +08:00
刘祥超
4096f11909 修复一处测试用例 2022-08-06 20:52:52 +08:00
刘祥超
5df209b6d5 优化代码 2022-08-04 11:34:06 +08:00
刘祥超
db353fe025 nftables封禁IP使用异步操作 2022-08-04 11:01:16 +08:00
刘祥超
30c3e143b8 edge-node pprof命令增加--addr参数 2022-08-04 10:18:23 +08:00
刘祥超
15fe7b33a4 新增屏蔽错误 2022-08-04 10:04:26 +08:00
刘祥超
17e0666ba4 优化代码 2022-08-03 23:31:08 +08:00
刘祥超
3f24bfaaf5 HTTP读取L2节点时增加最大连接数、最大空闲连接数、最大空闲时间 2022-08-03 20:54:08 +08:00
刘祥超
ceadcfece9 HTTP源站有多个L2节点时发生错误时会主动尝试下一个L2节点 2022-08-03 20:30:59 +08:00
刘祥超
bc706237ef HTTP源站读取错误时自动尝试下一个源站 2022-08-03 19:33:50 +08:00
刘祥超
b8c5a78f2e TCP/TLS/UDP第一次连接源站失败后,自动尝试下一个源站 2022-08-03 16:09:47 +08:00
刘祥超
9ad1c3a3c8 TLS支持默认SNI回源 2022-08-03 11:04:58 +08:00
刘祥超
8cfde43f5d TCP连接源站失败时关闭连接/TCP连接也支持备用源站 2022-08-01 19:28:27 +08:00
刘祥超
6f843b071a 执行IP名单更新任务时防止阻塞/优化节点升级代码 2022-08-01 18:54:54 +08:00
刘祥超
a72b025900 优化忽略客户端关闭连接错误条件 2022-08-01 16:53:08 +08:00
刘祥超
dfb66775d7 40x, 50x提示默认使用HTML;50x提示增加原因信息 2022-07-30 10:48:41 +08:00
刘祥超
c4dca2df30 UDP也记录带宽 2022-07-30 09:30:05 +08:00
刘祥超
a3525bdaa4 修复节点自动升级时无法自动启动的Bug 2022-07-28 14:38:08 +08:00
刘祥超
09390bbb97 优化代码 2022-07-27 18:10:43 +08:00
刘祥超
70ae4391d7 减少Daemon使用的内存 2022-07-26 09:41:43 +08:00
刘祥超
85931f55e1 修改版本号为0.4.11 2022-07-26 09:33:38 +08:00
刘祥超
0f5f03c9ed 取消IO保护 2022-07-26 08:29:22 +08:00
刘祥超
1181a0585b 优化IP名单相关代码/关闭、重启进程时自动关闭IP名单本地缓存数据库 2022-07-25 20:23:40 +08:00
刘祥超
fd6fa929de 修复连接数限制计算错误 2022-07-25 19:48:03 +08:00
刘祥超
73888c98a8 WAF多个相同Key的cc2统计规则不再重复累加 2022-07-25 09:34:34 +08:00
刘祥超
04ebfbea8a 节点状态中包含主程序位置 2022-07-21 15:07:12 +08:00
刘祥超
2b650fd285 API RPC配置增加disableUpdate,可以停用自动更新API节点 2022-07-21 14:06:38 +08:00
刘祥超
515a590681 地区封禁也可以使用自定义页面 2022-07-21 13:26:34 +08:00
刘祥超
62f9d1f09a 优化Firewalld添加端口方法,自动聚合连续的端口号 2022-07-21 11:53:23 +08:00
刘祥超
25c11f3d69 修改版本为v0.4.10 2022-07-20 18:14:12 +08:00
刘祥超
fde18c3b82 修复UserAgent中操作系统或浏览器版本中含有非UTF-8字符无法上传的问题 2022-07-20 16:12:46 +08:00
325 changed files with 10046 additions and 4221 deletions

View File

@@ -50,9 +50,9 @@ function build() {
fi
cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
cp "$ROOT"/configs/cluster.template.yaml "$DIST"/configs
cp -R "$ROOT"/www "$DIST"/
cp -R "$ROOT"/pages "$DIST"/
cp -R "$ROOT"/resources "$DIST"/
# we support TOA on linux/amd64 only
if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

View File

@@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"flag"
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/apps"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
@@ -19,12 +20,12 @@ import (
)
func main() {
app := apps.NewAppCmd().
var app = apps.NewAppCmd().
Version(teaconst.Version).
Product(teaconst.ProductName).
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog]").
Usage(teaconst.ProcessName + " [trackers|goman|conns|gc]").
Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove] IP")
Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove|ip.close] IP")
app.On("test", func() {
err := nodes.NewNode().Test()
@@ -67,24 +68,30 @@ func main() {
fmt.Println("done")
})
app.On("pprof", func() {
// TODO 自己指定端口
addr := "127.0.0.1:6060"
var flagSet = flag.NewFlagSet("pprof", flag.ExitOnError)
var addr string
flagSet.StringVar(&addr, "addr", "", "")
_ = flagSet.Parse(os.Args[2:])
if len(addr) == 0 {
addr = "127.0.0.1:6060"
}
logs.Println("starting with pprof '" + addr + "'...")
go func() {
err := http.ListenAndServe(addr, nil)
if err != nil {
logs.Println("[error]" + err.Error())
logs.Println("[ERROR]" + err.Error())
}
}()
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
app.On("dbstat", func() {
teaconst.EnableDBStat = true
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
app.On("trackers", func() {
@@ -154,7 +161,7 @@ func main() {
app.On("ip.drop", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS]")
fmt.Println("Usage: edge-node ip.drop IP [--timeout=SECONDS] [--async]")
return
}
var ip = args[0]
@@ -168,6 +175,11 @@ func main() {
if ok {
timeoutSeconds = types.Int(timeout[0])
}
var async = false
_, ok = options["async"]
if ok {
async = true
}
fmt.Println("drop ip '" + ip + "' for '" + types.String(timeoutSeconds) + "' seconds")
var sock = gosock.NewTmpSock(teaconst.ProcessName)
@@ -176,6 +188,7 @@ func main() {
Params: map[string]interface{}{
"ip": ip,
"timeoutSeconds": timeoutSeconds,
"async": async,
},
})
if err != nil {
@@ -228,6 +241,38 @@ func main() {
}
}
})
app.On("ip.close", func() {
var args = os.Args[2:]
if len(args) == 0 {
fmt.Println("Usage: edge-node ip.close IP")
return
}
var ip = args[0]
if len(net.ParseIP(ip)) == 0 {
fmt.Println("IP '" + ip + "' is invalid")
return
}
fmt.Println("close ip '" + ip)
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{
Code: "closeIP",
Params: map[string]any{
"ip": ip,
},
})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
} else {
var errString = maps.NewMap(reply.Params).GetString("error")
if len(errString) > 0 {
fmt.Println("[ERROR]" + errString)
} else {
fmt.Println("ok")
}
}
})
app.On("ip.remove", func() {
var args = os.Args[2:]
if len(args) == 0 {
@@ -305,8 +350,23 @@ func main() {
}
}
})
app.On("bandwidth", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
var statsMap = maps.NewMap(reply.Params).Get("stats")
statsJSON, err := json.MarshalIndent(statsMap, "", " ")
if err != nil {
fmt.Println("[ERROR]" + err.Error())
return
}
fmt.Println(string(statsJSON))
})
app.Run(func() {
node := nodes.NewNode()
var node = nodes.NewNode()
node.Start()
})
}

15
go.mod
View File

@@ -4,6 +4,7 @@ go 1.18
replace (
github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
github.com/fsnotify/fsnotify => github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4
)
require (
@@ -16,10 +17,10 @@ require (
github.com/go-redis/redis/v8 v8.11.5
github.com/golang/protobuf v1.5.2
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5
github.com/klauspost/compress v1.15.8
github.com/mattn/go-sqlite3 v1.14.9
github.com/mdlayher/netlink v1.4.2
@@ -27,20 +28,22 @@ require (
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
github.com/shirou/gopsutil/v3 v3.22.2
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
golang.org/x/text v0.3.7
google.golang.org/grpc v1.45.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
rogchap.com/v8go v0.7.0
)
require (
github.com/BurntSushi/toml v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/webp v1.1.0 // indirect
github.com/chai2010/webp v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect

84
go.sum
View File

@@ -1,17 +1,13 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670/go.mod h1:iRWAFbKXMMkVQyxZ1PfGlkBr1TjATx1zy2MRprV7A3Q=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -20,8 +16,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/webp v1.1.0 h1:4Ei0/BRroMF9FaXDG2e4OxwFcuW2vcXd+A6tyqTJUQQ=
github.com/chai2010/webp v1.1.0/go.mod h1:LP12PG5IFmLGHUU26tBiCBKnghxx3toZFwDjOYvd3Ow=
github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -37,7 +33,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f h1:q/DpyjJjZs94bziQ7YkBmIlpqbVP7yw179rnzoNVX1M=
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f/go.mod h1:QGrK8vMWWHQYQ3QU9bw9Y9OPNfxccGzfb41qjvVeXtY=
github.com/dgryski/go-rendezvous v0.0.0-20200624174652-8d2f3be8b2d9/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -48,10 +43,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -83,20 +76,20 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6 h1:btadZscaRmsi/+fOhkyUguRpSnrf6dykNEWxDeUCj9I=
github.com/google/nftables v0.0.0-20220407195405-950e408d48c6/go.mod h1:0F8on3JWMkm+xahTHItkiu/E1SPqMd0TOxNweQv8ptE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55 h1:shQNx0flJFBwKsGE7Hs3bI2bDz+YF0zl/4qE8B2KRiY=
github.com/iwind/TeaGo v0.0.0-20220807030847-31de8e1cbe55/go.mod h1:fi/Pq+/5m2HZoseM+39dMF57ANXRt6w4PkGu3NXPc5s=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4 h1:PKtXlgNHJhdwl5ozio7KRV3n0SckMw+8ZC2NCpRSv8U=
github.com/iwind/fsnotify v1.5.2-0.20220817040843-193be2051ff4/go.mod h1:DmAukmDY25inGlriLn0B2jidmvaDR1REOcPXwvzvDPI=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4 h1:VWGsCqTzObdlbf7UUE3oceIpcEKi4C/YBUszQXk118A=
github.com/iwind/gosock v0.0.0-20211103081026-ee4652210ca4/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8 h1:AojsHz9Es9B3He2MQQxeRq3TyD//o9huxUo7r1wh44g=
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8/go.mod h1:QJBY2txYhLMzwLO29iB5ujDJ3s3V7DsZ582nw4Ss+tM=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5 h1:SgKhrqRhWVpu0ZKxQM8MGjdhy7hlH3Xax8snwsZWSic=
github.com/iwind/gowebp v0.0.0-20220819053541-c235395277b5/go.mod h1:Re7TEhwL+ygnxFg52fC0PWy01ULAIZp2QR0q5WwEOQA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 h1:uhL5Gw7BINiiPAo24A2sxkcDI0Jt/sqp1v5xQCniEFA=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -109,14 +102,10 @@ github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e h1:LvL4XsI70QxOGHed6yhQtAU34Kx3Qq2wwBzGFKY8zKk=
github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e/go.mod h1:kLgvv7o6UM+0QSf0QjAse3wReFDsb9qbZJdfexWlrQw=
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -150,23 +139,11 @@ github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb h1:2dC7L10LmTqlyMV
github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldOGkPw=
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -180,9 +157,7 @@ github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtS
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
@@ -196,41 +171,30 @@ github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -259,21 +223,15 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -300,11 +258,10 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -316,7 +273,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
@@ -330,7 +286,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
@@ -339,7 +294,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
@@ -358,25 +312,19 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
rogchap.com/v8go v0.7.0 h1:kgjbiO4zE5itA962ze6Hqmbs4HgZbGzmueCXsZtremg=
rogchap.com/v8go v0.7.0/go.mod h1:MxgP3pL2MW4dpme/72QRs8sgNMmM0pRc8DPhcuLWPAs=

View File

@@ -41,7 +41,7 @@ func (this *LogWriter) Init() {
this.c = make(chan string, 1024)
// 异步写入文件
var maxFileSize = 2 * sizes.G // 文件最大尺寸,超出此尺寸则清空
var maxFileSize = 128 * sizes.M // 文件最大尺寸,超出此尺寸则清空
if fp != nil {
goman.New(func() {
var totalSize int64 = 0
@@ -89,7 +89,10 @@ func (this *LogWriter) Write(message string) {
}
}
this.c <- message
select {
case this.c <- message:
default:
}
}
func (this *LogWriter) Close() {

11
internal/apps/main.go Normal file
View File

@@ -0,0 +1,11 @@
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package apps
import teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
func RunMain(f func()) {
if teaconst.IsMain {
f()
}
}

View File

@@ -3,6 +3,7 @@
package caches
const (
SuffixAll = "@GOEDGE_" // 通用后缀
SuffixWebP = "@GOEDGE_WEBP" // WebP后缀
SuffixCompression = "@GOEDGE_" // 压缩后缀 SuffixCompression + Encoding
SuffixMethod = "@GOEDGE_" // 请求方法后缀 SuffixMethod + RequestMethod

View File

@@ -0,0 +1,11 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches
import "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
type FileDir struct {
Path string
Capacity *shared.SizeCapacity
IsFull bool
}

View File

@@ -2,6 +2,7 @@ package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"strings"
"time"
)
@@ -59,3 +60,17 @@ func (this *Item) IncreaseHit(week int32) {
this.Week = week
}
}
func (this *Item) RequestURI() string {
var schemeIndex = strings.Index(this.Key, "://")
if schemeIndex <= 0 {
return ""
}
var firstSlashIndex = strings.Index(this.Key[schemeIndex+3:], "/")
if firstSlashIndex <= 0 {
return ""
}
return this.Key[schemeIndex+3+firstSlashIndex:]
}

View File

@@ -81,3 +81,14 @@ func TestItems_Memory2(t *testing.T) {
t.Log(w, len(i))
}
}
func TestItem_RequestURI(t *testing.T) {
for _, u := range []string{
"https://goedge.cn/hello/world",
"https://goedge.cn:8080/hello/world",
"https://goedge.cn/hello/world?v=1&t=123",
} {
var item = &Item{Key: u}
t.Log(u, "=>", item.RequestURI())
}
}

View File

@@ -7,9 +7,9 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
"github.com/iwind/TeaGo/types"
_ "github.com/mattn/go-sqlite3"
"os"
"sync/atomic"
"time"
@@ -96,13 +96,13 @@ func (this *FileList) Reset() error {
}
func (this *FileList) Add(hash string, item *Item) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
err := db.Add(hash, item)
err := db.AddAsync(hash, item)
if err != nil {
return err
}
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
}
func (this *FileList) Exist(hash string) (bool, error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// 如果Hash列表里不存在那么必然不存在
if !db.hashMap.Exist(hash) {
return false, nil
}
var item = this.memoryCache.Read(hash)
if item != nil {
return true, nil
@@ -155,6 +160,7 @@ func (this *FileList) CleanPrefix(prefix string) error {
}
defer func() {
// TODO 需要优化
this.memoryCache.Clean()
}()
@@ -167,6 +173,46 @@ func (this *FileList) CleanPrefix(prefix string) error {
return nil
}
// CleanMatchKey 清理通配符匹配的缓存数据,类似于 https://*.example.com/hello
func (this *FileList) CleanMatchKey(key string) error {
if len(key) == 0 {
return nil
}
defer func() {
// TODO 需要优化
this.memoryCache.Clean()
}()
for _, db := range this.dbList {
err := db.CleanMatchKey(key)
if err != nil {
return err
}
}
return nil
}
// CleanMatchPrefix 清理通配符匹配的缓存数据,类似于 https://*.example.com/prefix/
func (this *FileList) CleanMatchPrefix(prefix string) error {
if len(prefix) == 0 {
return nil
}
defer func() {
// TODO 需要优化
this.memoryCache.Clean()
}()
for _, db := range this.dbList {
err := db.CleanMatchPrefix(prefix)
if err != nil {
return err
}
}
return nil
}
func (this *FileList) Remove(hash string) error {
_, err := this.remove(hash)
return err
@@ -225,9 +271,9 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
return err
}
if notFound {
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return err
return db.WrapError(err)
}
}
@@ -266,7 +312,7 @@ func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
_ = check
row := db.statStmt.QueryRow()
var row = db.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
@@ -291,13 +337,13 @@ func (this *FileList) Count() (int64, error) {
// IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return nil
}
return db.IncreaseHit(hash)
return db.IncreaseHitAsync(hash)
}
// OnAdd 添加事件
@@ -326,17 +372,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
return fnv.HashString(hash) % CountFileDB
}
func (this *FileList) getDB(hash string) *FileListDB {
func (this *FileList) GetDB(hash string) *FileListDB {
return this.dbList[fnv.HashString(hash)%CountFileDB]
}
func (this *FileList) remove(hash string) (notFound bool, err error) {
var db = this.getDB(hash)
var db = this.GetDB(hash)
if !db.IsReady() {
return false, nil
}
// HashMap中不存在则确定不存在
if !db.hashMap.Exist(hash) {
return true, nil
}
defer db.hashMap.Delete(hash)
// 从缓存中删除
this.memoryCache.Delete(hash)
@@ -357,16 +409,16 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
return false, err
}
_, err = db.deleteByHashStmt.Exec(hash)
err = db.DeleteAsync(hash)
if err != nil {
return false, err
return false, db.WrapError(err)
}
atomic.AddInt64(&this.total, -1)
_, err = db.deleteHitByHashStmt.Exec(hash)
err = db.DeleteHitAsync(hash)
if err != nil {
return false, err
return false, db.WrapError(err)
}
if this.onRemove != nil {
@@ -398,7 +450,7 @@ func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
remotelogs.Println("CACHE", "upgrading local database finished")
}()
db, err := sql.Open("sqlite3", "file:"+indexDBPath+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
db, err := dbs.OpenWriter("file:" + indexDBPath + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
if err != nil {
return err
}

View File

@@ -3,22 +3,34 @@
package caches
import (
"database/sql"
"errors"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"net"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
"time"
)
type FileListDB struct {
dbPath string
readDB *dbs.DB
writeDB *dbs.DB
writeBatch *dbs.Batch
hashMap *FileListHashMap
itemsTableName string
hitsTableName string
@@ -28,57 +40,98 @@ type FileListDB struct {
isReady bool
// cacheItems
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
selectByHashStmt *dbs.Stmt // 使用hash查询数据
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
statStmt *dbs.Stmt // 统计
purgeStmt *dbs.Stmt // 清理
deleteAllStmt *dbs.Stmt // 删除所有数据
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
insertSQL string
selectByHashStmt *dbs.Stmt // 使用hash查询数据
selectHashListStmt *dbs.Stmt
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
deleteByHashSQL string
statStmt *dbs.Stmt // 统计
purgeStmt *dbs.Stmt // 清理
deleteAllStmt *dbs.Stmt // 删除所有数据
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
updateAccessWeekSQL string // 修改访问日期
// hits
insertHitStmt *dbs.Stmt // 写入数据
increaseHitStmt *dbs.Stmt // 增加点击量
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
insertHitSQL string // 写入数据
increaseHitSQL string // 增加点击量
deleteHitByHashSQL string // 根据hash删除数据
}
func NewFileListDB() *FileListDB {
return &FileListDB{}
return &FileListDB{
hashMap: NewFileListHashMap(),
}
}
func (this *FileListDB) Open(dbPath string) error {
this.dbPath = dbPath
// 动态调整Cache值
var cacheSize = 32000
var memoryGB = utils.SystemMemoryGB()
if memoryGB >= 8 {
cacheSize += 32000 * memoryGB / 8
}
// write db
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
// 这里不能加 EXCLUSIVE 锁,不然异步事务可能会失败
writeDB, err := dbs.OpenWriter("file:" + dbPath + "?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=" + types.String(cacheSize) + "&_secure_delete=FAST")
if err != nil {
return errors.New("open write database failed: " + err.Error())
}
writeDB.SetMaxOpenConns(1)
this.writeDB = writeDB
// TODO 耗时过长,暂时不整理数据库
// TODO 需要根据行数来判断是否VACUUM
// TODO 注意VACUUM反而可能让数据库文件变大
/**_, err = db.Exec("VACUUM")
if err != nil {
return err
}**/
this.writeDB = dbs.NewDB(writeDB)
// 检查是否损坏
// TODO 暂时屏蔽,因为用时过长
var recoverEnv, _ = os.LookupEnv("EdgeRecover")
if len(recoverEnv) > 0 && this.shouldRecover() {
for _, indexName := range []string{"staleAt", "hash"} {
_, _ = this.writeDB.Exec(`REINDEX "` + indexName + `"`)
}
}
this.writeBatch = dbs.NewBatch(writeDB.RawDB(), 4)
this.writeBatch.OnFail(func(err error) {
remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error()+" ("+filepath.Base(this.dbPath)+")")
})
goman.New(func() {
this.writeBatch.Exec()
})
if teaconst.EnableDBStat {
this.writeBatch.EnableStat(true)
this.writeDB.EnableStat(true)
}
// read db
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
readDB, err := dbs.OpenReader("file:" + dbPath + "?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=" + types.String(cacheSize))
if err != nil {
return errors.New("open read database failed: " + err.Error())
}
readDB.SetMaxOpenConns(runtime.NumCPU())
this.readDB = dbs.NewDB(readDB)
this.readDB = readDB
if teaconst.EnableDBStat {
this.readDB.EnableStat(true)
@@ -115,7 +168,8 @@ func (this *FileListDB) Init() error {
return err
}
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt", "accessWeek") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
if err != nil {
return err
}
@@ -125,7 +179,13 @@ func (this *FileListDB) Init() error {
return err
}
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
if err != nil {
return err
}
this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
if err != nil {
return err
}
@@ -145,27 +205,29 @@ func (this *FileListDB) Init() error {
return err
}
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
if err != nil {
return err
}
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.updateAccessWeekSQL = `UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
return err
}
this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
this.isReady = true
// 加载HashMap
go func() {
err := this.hashMap.Load(this)
if err != nil {
remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
}
}()
return nil
}
@@ -177,17 +239,47 @@ func (this *FileListDB) Total() int64 {
return this.total
}
func (this *FileListDB) Add(hash string, item *Item) error {
func (this *FileListDB) AddAsync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
// 放入队列
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime(), timeutil.Format("YW"))
return nil
}
func (this *FileListDB) AddSync(hash string, item *Item) error {
this.hashMap.Add(hash)
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime(), timeutil.Format("YW"))
if err != nil {
return this.WrapError(err)
}
return nil
}
func (this *FileListDB) DeleteAsync(hash string) error {
this.hashMap.Delete(hash)
this.writeBatch.Add(this.deleteByHashSQL, hash)
return nil
}
func (this *FileListDB) DeleteSync(hash string) error {
this.hashMap.Delete(hash)
_, err := this.deleteByHashStmt.Exec(hash)
if err != nil {
return err
}
return nil
}
@@ -228,28 +320,56 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
count = 100
}
hashList, err = this.listLFUItems(count)
// 先找过期的
hashList, err = this.ListExpiredItems(count)
if err != nil {
return
}
var l = len(hashList)
if len(hashList) > count/2 {
return
// 从旧缓存中补充
if l < count {
oldHashList, err := this.listOlderItems(count - l)
if err != nil {
return nil, err
}
hashList = append(hashList, oldHashList...)
}
// 不足补齐
olderHashList, err := this.listOlderItems(count - len(hashList))
return hashList, nil
}
func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
rows, err := this.selectHashListStmt.Query(lastId)
if err != nil {
return nil, err
return nil, 0, err
}
hashList = append(hashList, olderHashList...)
var id int64
var hash string
for rows.Next() {
err = rows.Scan(&id, &hash)
if err != nil {
_ = rows.Close()
return
}
maxId = id
hashList = append(hashList, hash)
}
_ = rows.Close()
return
}
func (this *FileListDB) IncreaseHit(hash string) error {
func (this *FileListDB) IncreaseHitAsync(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return err
this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
this.writeBatch.Add(this.updateAccessWeekSQL, week, hash)
return nil
}
func (this *FileListDB) DeleteHitAsync(hash string) error {
this.writeBatch.Add(this.deleteHitByHashSQL, hash)
return nil
}
func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -262,7 +382,7 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
for {
result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
if err != nil {
return err
return this.WrapError(err)
}
affectedRows, err := result.RowsAffected()
if err != nil {
@@ -274,6 +394,85 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
}
}
func (this *FileListDB) CleanMatchKey(key string) error {
if !this.isReady {
return nil
}
// 忽略 @GOEDGE_
if strings.Contains(key, SuffixAll) {
return nil
}
u, err := url.Parse(key)
if err != nil {
return nil
}
var host = u.Host
hostPart, _, err := net.SplitHostPort(host)
if err == nil && len(hostPart) > 0 {
host = hostPart
}
if len(host) == 0 {
return nil
}
// 转义
var queryKey = strings.ReplaceAll(key, "%", "\\%")
queryKey = strings.ReplaceAll(queryKey, "_", "\\_")
queryKey = strings.Replace(queryKey, "*", "%", 1)
// TODO 检查大批量数据下的操作性能
var staleLife = 600 // TODO 需要可以设置
var unixTime = utils.UnixTime() // 只删除当前的,不删除新的
_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey)
if err != nil {
return err
}
_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey+SuffixAll+"%")
if err != nil {
return err
}
return nil
}
func (this *FileListDB) CleanMatchPrefix(prefix string) error {
if !this.isReady {
return nil
}
u, err := url.Parse(prefix)
if err != nil {
return nil
}
var host = u.Host
hostPart, _, err := net.SplitHostPort(host)
if err == nil && len(hostPart) > 0 {
host = hostPart
}
if len(host) == 0 {
return nil
}
// 转义
var queryPrefix = strings.ReplaceAll(prefix, "%", "\\%")
queryPrefix = strings.ReplaceAll(queryPrefix, "_", "\\_")
queryPrefix = strings.Replace(queryPrefix, "*", "%", 1)
queryPrefix += "%"
// TODO 检查大批量数据下的操作性能
var staleLife = 600 // TODO 需要可以设置
var unixTime = utils.UnixTime() // 只删除当前的,不删除新的
_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryPrefix)
return err
}
func (this *FileListDB) CleanAll() error {
if !this.isReady {
return nil
@@ -281,9 +480,11 @@ func (this *FileListDB) CleanAll() error {
_, err := this.deleteAllStmt.Exec()
if err != nil {
return err
return this.WrapError(err)
}
this.hashMap.Clean()
return nil
}
@@ -300,6 +501,9 @@ func (this *FileListDB) Close() error {
if this.selectByHashStmt != nil {
_ = this.selectByHashStmt.Close()
}
if this.selectHashListStmt != nil {
_ = this.selectHashListStmt.Close()
}
if this.deleteByHashStmt != nil {
_ = this.deleteByHashStmt.Close()
}
@@ -316,17 +520,8 @@ func (this *FileListDB) Close() error {
_ = this.listOlderItemsStmt.Close()
}
if this.insertHitStmt != nil {
_ = this.insertHitStmt.Close()
}
if this.increaseHitStmt != nil {
_ = this.increaseHitStmt.Close()
}
if this.deleteHitByHashStmt != nil {
_ = this.deleteHitByHashStmt.Close()
}
if this.lfuHitsStmt != nil {
_ = this.lfuHitsStmt.Close()
if this.writeBatch != nil {
this.writeBatch.Close()
}
var errStrings []string
@@ -351,11 +546,19 @@ func (this *FileListDB) Close() error {
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
}
func (this *FileListDB) WrapError(err error) error {
if err == nil {
return nil
}
return errors.New(err.Error() + "(file: " + this.dbPath + ")")
}
// 初始化
func (this *FileListDB) initTables(times int) error {
{
// expiredAt - 过期时间,用来判断有无过期
// staleAt - 过时缓存最大时间,用来清理缓存
// 不对 hash 增加 unique 参数,是尽可能避免产生 malformed 错误
_, err := this.writeDB.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
@@ -368,7 +571,8 @@ func (this *FileListDB) initTables(times int) error {
"staleAt" integer DEFAULT 0,
"createdAt" integer DEFAULT 0,
"host" varchar(128),
"serverId" integer
"serverId" integer,
"accessWeek" varchar(6)
);
DROP INDEX IF EXISTS "createdAt";
@@ -380,23 +584,32 @@ ON "` + this.itemsTableName + `" (
"staleAt" ASC
);
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
CREATE INDEX IF NOT EXISTS "hash"
ON "` + this.itemsTableName + `" (
"hash" ASC
);
ALTER TABLE "cacheItems" ADD "accessWeek" varchar(6);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(times + 1)
}
return err
// 忽略可以预期的错误
if strings.Contains(err.Error(), "duplicate column name") {
err = nil
}
return err
// 尝试删除重建
if err != nil {
if times < 3 {
_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(times + 1)
}
return this.WrapError(err)
}
return this.WrapError(err)
}
}
}
@@ -421,37 +634,16 @@ ON "` + this.hitsTableName + `" (
if dropErr == nil {
return this.initTables(times + 1)
}
return err
return this.WrapError(err)
}
return err
return this.WrapError(err)
}
}
return nil
}
func (this *FileListDB) listLFUItems(count int) (hashList []string, err error) {
rows, err := this.lfuHitsStmt.Query(count)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
return nil, err
}
hashList = append(hashList, hash)
}
return hashList, nil
}
func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) {
rows, err := this.listOlderItemsStmt.Query(count)
if err != nil {
@@ -472,3 +664,21 @@ func (this *FileListDB) listOlderItems(count int) (hashList []string, err error)
return hashList, nil
}
func (this *FileListDB) shouldRecover() bool {
result, err := this.writeDB.Query("pragma integrity_check;")
if err != nil {
logs.Println(result)
}
var errString = ""
var shouldRecover = false
for result.Next() {
err = result.Scan(&errString)
if strings.TrimSpace(errString) != "ok" {
shouldRecover = true
}
break
}
_ = result.Close()
return shouldRecover
}

View File

@@ -0,0 +1,87 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"testing"
"time"
)
func TestFileListDB_ListLFUItems(t *testing.T) {
var db = caches.NewFileListDB()
err := db.Open(Tea.Root + "/data/cache-db-large.db")
//err := db.Open(Tea.Root + "/data/cache-index/p1/db-0.db")
if err != nil {
t.Fatal(err)
}
err = db.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = db.Close()
}()
hashList, err := db.ListLFUItems(100)
if err != nil {
t.Fatal(err)
}
t.Log("[", len(hashList), "]", hashList)
}
func TestFileListDB_IncreaseHitAsync(t *testing.T) {
var db = caches.NewFileListDB()
err := db.Open(Tea.Root + "/data/cache-db-large.db")
if err != nil {
t.Fatal(err)
}
err = db.Init()
err = db.IncreaseHitAsync("4598e5231ba47d6ec7aa9ea640ff2eaf")
if err != nil {
t.Fatal(err)
}
// wait transaction
time.Sleep(1 * time.Second)
}
func TestFileListDB_CleanMatchKey(t *testing.T) {
var db = caches.NewFileListDB()
err := db.Open(Tea.Root + "/data/cache-db-large.db")
if err != nil {
t.Fatal(err)
}
err = db.Init()
err = db.CleanMatchKey("https://*.goedge.cn/large-text")
if err != nil {
t.Fatal(err)
}
err = db.CleanMatchKey("https://*.goedge.cn:1234/large-text?%2B____")
if err != nil {
t.Fatal(err)
}
}
func TestFileListDB_CleanMatchPrefix(t *testing.T) {
var db = caches.NewFileListDB()
err := db.Open(Tea.Root + "/data/cache-db-large.db")
if err != nil {
t.Fatal(err)
}
err = db.Init()
err = db.CleanMatchPrefix("https://*.goedge.cn/large-text")
if err != nil {
t.Fatal(err)
}
err = db.CleanMatchPrefix("https://*.goedge.cn:1234/large-text?%2B____")
if err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,120 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"math/big"
"sync"
)
// FileListHashMap 文件Hash列表
type FileListHashMap struct {
m map[uint64]zero.Zero
locker sync.RWMutex
isAvailable bool
isReady bool
}
func NewFileListHashMap() *FileListHashMap {
return &FileListHashMap{
m: map[uint64]zero.Zero{},
isAvailable: false,
isReady: false,
}
}
func (this *FileListHashMap) Load(db *FileListDB) error {
// 如果系统内存过小,我们不缓存
if utils.SystemMemoryGB() < 3 {
return nil
}
this.isAvailable = true
var lastId int64
for {
hashList, maxId, err := db.ListHashes(lastId)
if err != nil {
return err
}
if len(hashList) == 0 {
break
}
this.AddHashes(hashList)
lastId = maxId
}
this.isReady = true
return nil
}
func (this *FileListHashMap) Add(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
this.m[this.bigInt(hash)] = zero.New()
this.locker.Unlock()
}
func (this *FileListHashMap) AddHashes(hashes []string) {
if !this.isAvailable {
return
}
this.locker.Lock()
for _, hash := range hashes {
this.m[this.bigInt(hash)] = zero.New()
}
this.locker.Unlock()
}
func (this *FileListHashMap) Delete(hash string) {
if !this.isAvailable {
return
}
this.locker.Lock()
delete(this.m, this.bigInt(hash))
this.locker.Unlock()
}
func (this *FileListHashMap) Exist(hash string) bool {
if !this.isAvailable {
return true
}
if !this.isReady {
// 只有完全Ready时才能判断是否为false
return true
}
this.locker.RLock()
_, ok := this.m[this.bigInt(hash)]
this.locker.RUnlock()
return ok
}
func (this *FileListHashMap) Clean() {
this.locker.Lock()
this.m = map[uint64]zero.Zero{}
this.locker.Unlock()
}
func (this *FileListHashMap) IsReady() bool {
return this.isReady
}
func (this *FileListHashMap) Len() int {
this.locker.Lock()
defer this.locker.Unlock()
return len(this.m)
}
func (this *FileListHashMap) bigInt(hash string) uint64 {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
return bigInt.Uint64()
}

View File

@@ -0,0 +1,96 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"math/big"
"runtime"
"strconv"
"testing"
"time"
)
func TestFileListHashMap_Memory(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = caches.NewFileListHashMap()
for i := 0; i < 1_000_000; i++ {
m.Add(stringutil.Md5(types.String(i)))
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_Memory2(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
var m = map[uint64]zero.Zero{}
for i := 0; i < 1_000_000; i++ {
m[uint64(i)] = zero.New()
}
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
}
func TestFileListHashMap_BigInt(t *testing.T) {
for _, s := range []string{"1", "2", "3", "123", "123456"} {
var hash = stringutil.Md5(s)
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
}
}
func TestFileListHashMap_Load(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
var m = caches.NewFileListHashMap()
var before = time.Now()
var db = list.GetDB("abc")
err = m.Load(db)
if err != nil {
t.Fatal(err)
}
t.Log(time.Since(before).Seconds()*1000, "ms")
t.Log("count:", m.Len())
m.Add("abc")
for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
t.Log(hash, "=>", m.Exist(hash))
}
}
func Benchmark_BigInt(b *testing.B) {
var hash = stringutil.Md5("123456")
b.ResetTimer()
for i := 0; i < b.N; i++ {
var bigInt = big.NewInt(0)
bigInt.SetString(hash, 16)
_ = bigInt.Uint64()
}
}

View File

@@ -69,8 +69,8 @@ func TestFileList_Add_Many(t *testing.T) {
_ = list.Close()
}()
before := time.Now()
for i := 0; i < 100_000; i++ {
var before = time.Now()
for i := 0; i < 10_000_000; i++ {
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &caches.Item{
Key: u,
@@ -290,12 +290,12 @@ func TestFileList_Stat(t *testing.T) {
}
func TestFileList_Count(t *testing.T) {
list := caches.NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
var before = time.Now()
count, err := list.Count()
if err != nil {
t.Fatal(err)
@@ -367,6 +367,7 @@ func BenchmarkFileList_Exist(b *testing.B) {
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
}

View File

@@ -18,6 +18,12 @@ type ListInterface interface {
// CleanPrefix 清除某个前缀的缓存
CleanPrefix(prefix string) error
// CleanMatchKey 清除通配符匹配的Key
CleanMatchKey(key string) error
// CleanMatchPrefix 清除通配符匹配的前缀
CleanMatchPrefix(prefix string) error
// Remove 删除内容
Remove(hash string) error

View File

@@ -1,8 +1,11 @@
package caches
import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/logs"
"net"
"net/url"
"strconv"
"strings"
"sync"
@@ -146,6 +149,82 @@ func (this *MemoryList) CleanPrefix(prefix string) error {
return nil
}
// CleanMatchKey 清理通配符匹配的缓存数据,类似于 https://*.example.com/hello
func (this *MemoryList) CleanMatchKey(key string) error {
if strings.Contains(key, SuffixAll) {
return nil
}
u, err := url.Parse(key)
if err != nil {
return nil
}
var host = u.Host
hostPart, _, err := net.SplitHostPort(host)
if err == nil && len(hostPart) > 0 {
host = hostPart
}
if len(host) == 0 {
return nil
}
var requestURI = u.RequestURI()
this.locker.RLock()
defer this.locker.RUnlock()
// TODO 需要优化性能支持千万级数据低于1s的处理速度
for _, itemMap := range this.itemMaps {
for _, item := range itemMap {
if configutils.MatchDomain(host, item.Host) {
var itemRequestURI = item.RequestURI()
if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) {
item.ExpiredAt = 0
}
}
}
}
return nil
}
// CleanMatchPrefix 清理通配符匹配的缓存数据,类似于 https://*.example.com/prefix/
func (this *MemoryList) CleanMatchPrefix(prefix string) error {
u, err := url.Parse(prefix)
if err != nil {
return nil
}
var host = u.Host
hostPart, _, err := net.SplitHostPort(host)
if err == nil && len(hostPart) > 0 {
host = hostPart
}
if len(host) == 0 {
return nil
}
var requestURI = u.RequestURI()
var isRootPath = requestURI == "/"
this.locker.RLock()
defer this.locker.RUnlock()
// TODO 需要优化性能支持千万级数据低于1s的处理速度
for _, itemMap := range this.itemMaps {
for _, item := range itemMap {
if configutils.MatchDomain(host, item.Host) {
var itemRequestURI = item.RequestURI()
if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) {
item.ExpiredAt = 0
}
}
}
}
return nil
}
func (this *MemoryList) Remove(hash string) error {
this.locker.Lock()

View File

@@ -3,10 +3,10 @@ package caches
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"strconv"
"sync"
@@ -15,8 +15,12 @@ import (
var SharedManager = NewManager()
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventQuit, func() {
logs.Println("CACHE", "quiting cache manager")
remotelogs.Println("CACHE", "quiting cache manager")
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
})
}
@@ -25,7 +29,8 @@ func init() {
type Manager struct {
// 全局配置
MaxDiskCapacity *shared.SizeCapacity
DiskDir string
MainDiskDir string
SubDiskDirs []*serverconfigs.CacheDir
MaxMemoryCapacity *shared.SizeCapacity
policyMap map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
@@ -48,12 +53,10 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
this.locker.Lock()
defer this.locker.Unlock()
newPolicyIds := []int64{}
var newPolicyIds = []int64{}
for _, policy := range newPolicies {
// 使用节点单独的缓存目录
if len(this.DiskDir) > 0 {
policy.UpdateDiskDir(this.DiskDir)
}
policy.UpdateDiskDir(this.MainDiskDir, this.SubDiskDirs)
newPolicyIds = append(newPolicyIds, policy.Id)
}

View File

@@ -19,7 +19,7 @@ type OpenFileCache struct {
poolList *linkedlist.List
watcher *fsnotify.Watcher
locker sync.Mutex
locker sync.RWMutex
maxSize int
count int
@@ -54,13 +54,18 @@ func NewOpenFileCache(maxSize int) (*OpenFileCache, error) {
}
func (this *OpenFileCache) Get(filename string) *OpenFile {
this.locker.Lock()
defer this.locker.Unlock()
this.locker.RLock()
pool, ok := this.poolMap[filename]
this.locker.RUnlock()
if ok {
file, consumed := pool.Get()
if consumed {
this.locker.Lock()
this.count--
// pool如果为空也不需要从列表中删除避免put时需要重新创建
this.locker.Unlock()
}
return file
}
@@ -124,6 +129,9 @@ func (this *OpenFileCache) Close(filename string) {
pool, ok := this.poolMap[filename]
if ok {
// 设置关闭状态
pool.SetClosing()
delete(this.poolMap, filename)
this.poolList.Remove(pool.linkItem)
_ = this.watcher.Remove(filename)
@@ -146,6 +154,7 @@ func (this *OpenFileCache) CloseAll() {
this.poolMap = map[string]*OpenFilePool{}
this.poolList.Reset()
_ = this.watcher.Close()
this.count = 0
this.locker.Unlock()
}

View File

@@ -0,0 +1,43 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"testing"
"time"
)
func TestNewOpenFileCache_Close(t *testing.T) {
cache, err := caches.NewOpenFileCache(1024)
if err != nil {
t.Fatal(err)
}
cache.Debug()
cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Get("b.txt")
cache.Get("d.txt")
cache.Close("a.txt")
time.Sleep(100 * time.Second)
}
func TestNewOpenFileCache_CloseAll(t *testing.T) {
cache, err := caches.NewOpenFileCache(1024)
if err != nil {
t.Fatal(err)
}
cache.Debug()
cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
cache.Get("b.txt")
cache.Get("d.txt")
cache.CloseAll()
time.Sleep(6 * time.Second)
}

View File

@@ -12,6 +12,7 @@ type OpenFilePool struct {
linkItem *linkedlist.Item
filename string
version int64
isClosed bool
}
func NewOpenFilePool(filename string) *OpenFilePool {
@@ -29,26 +30,43 @@ func (this *OpenFilePool) Filename() string {
}
func (this *OpenFilePool) Get() (*OpenFile, bool) {
// 如果已经关闭,直接返回
if this.isClosed {
return nil, false
}
select {
case file := <-this.c:
err := file.SeekStart()
if err != nil {
_ = file.Close()
return nil, true
}
file.version = this.version
if file != nil {
err := file.SeekStart()
if err != nil {
_ = file.Close()
return nil, true
}
file.version = this.version
return file, true
return file, true
}
return nil, false
default:
return nil, false
}
}
func (this *OpenFilePool) Put(file *OpenFile) bool {
// 如果已关闭,则不接受新的文件
if this.isClosed {
_ = file.Close()
return false
}
// 检查文件版本号
if this.version > 0 && file.version > 0 && file.version != this.version {
_ = file.Close()
return false
}
// 加入Pool
select {
case this.c <- file:
return true
@@ -63,14 +81,18 @@ func (this *OpenFilePool) Len() int {
return len(this.c)
}
func (this *OpenFilePool) SetClosing() {
this.isClosed = true
}
func (this *OpenFilePool) Close() {
Loop:
this.isClosed = true
for {
select {
case file := <-this.c:
_ = file.Close()
default:
break Loop
return
}
}
}

View File

@@ -4,6 +4,8 @@ package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/rands"
"sync"
"testing"
)
@@ -15,3 +17,30 @@ func TestOpenFilePool_Get(t *testing.T) {
t.Log(pool.Get())
t.Log(pool.Get())
}
func TestOpenFilePool_Close(t *testing.T) {
var pool = caches.NewOpenFilePool("a")
pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
pool.Close()
}
func TestOpenFilePool_Concurrent(t *testing.T) {
var pool = caches.NewOpenFilePool("a")
var concurrent = 1000
var wg = &sync.WaitGroup{}
wg.Add(concurrent)
for i := 0; i < concurrent; i++ {
go func() {
defer wg.Done()
if rands.Int(0, 1) == 1 {
pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
}
if rands.Int(0, 1) == 0 {
pool.Get()
}
}()
}
wg.Wait()
}

View File

@@ -3,38 +3,88 @@
package caches
import (
"bytes"
"encoding/json"
"errors"
"io/ioutil"
"github.com/iwind/TeaGo/types"
"os"
"strconv"
)
// PartialRanges 内容分区范围定义
type PartialRanges struct {
Ranges [][2]int64 `json:"ranges"`
Version int `json:"version"` // 版本号
Ranges [][2]int64 `json:"ranges"` // 范围
BodySize int64 `json:"bodySize"` // 总长度
}
// NewPartialRanges 获取新对象
func NewPartialRanges() *PartialRanges {
return &PartialRanges{Ranges: [][2]int64{}}
func NewPartialRanges(expiresAt int64) *PartialRanges {
return &PartialRanges{
Ranges: [][2]int64{},
Version: 1,
}
}
// NewPartialRangesFromData 从数据中解析范围
func NewPartialRangesFromData(data []byte) (*PartialRanges, error) {
var rs = NewPartialRanges(0)
for {
var index = bytes.IndexRune(data, '\n')
if index < 0 {
break
}
var line = data[:index]
var colonIndex = bytes.IndexRune(line, ':')
if colonIndex > 0 {
switch string(line[:colonIndex]) {
case "v": // 版本号
rs.Version = types.Int(line[colonIndex+1:])
case "b": // 总长度
rs.BodySize = types.Int64(line[colonIndex+1:])
case "r": // 范围信息
var commaIndex = bytes.IndexRune(line, ',')
if commaIndex > 0 {
rs.Ranges = append(rs.Ranges, [2]int64{types.Int64(line[colonIndex+1 : commaIndex]), types.Int64(line[commaIndex+1:])})
}
}
}
data = data[index+1:]
if len(data) == 0 {
break
}
}
return rs, nil
}
// NewPartialRangesFromJSON 从JSON中解析范围
func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
var rs = NewPartialRanges()
var rs = NewPartialRanges(0)
err := json.Unmarshal(data, &rs)
if err != nil {
return nil, err
}
rs.Version = 0
return rs, nil
}
// NewPartialRangesFromFile 从文件中加载范围信息
func NewPartialRangesFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return NewPartialRangesFromJSON(data)
if len(data) == 0 {
return NewPartialRanges(0), nil
}
// 兼容老的JSON格式
if data[0] == '{' {
return NewPartialRangesFromJSON(data)
}
// 新的格式
return NewPartialRangesFromData(data)
}
// Add 添加新范围
@@ -105,29 +155,27 @@ func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool)
return
}
// AsJSON 转换为JSON
func (this *PartialRanges) AsJSON() ([]byte, error) {
return json.Marshal(this)
// 转换为字符串
func (this *PartialRanges) String() string {
var s = "v:" + strconv.Itoa(this.Version) + "\n" + // version
"b:" + this.formatInt64(this.BodySize) + "\n" // bodySize
for _, r := range this.Ranges {
s += "r:" + this.formatInt64(r[0]) + "," + this.formatInt64(r[1]) + "\n" // range
}
return s
}
// Bytes 将内容转换为字节
func (this *PartialRanges) Bytes() []byte {
return []byte(this.String())
}
// WriteToFile 写入到文件中
func (this *PartialRanges) WriteToFile(path string) error {
data, err := this.AsJSON()
if err != nil {
return errors.New("convert to json failed: " + err.Error())
}
return ioutil.WriteFile(path, data, 0666)
}
// ReadFromFile 从文件中读取
func (this *PartialRanges) ReadFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return NewPartialRangesFromJSON(data)
return os.WriteFile(path, this.Bytes(), 0666)
}
// Max 获取最大位置
func (this *PartialRanges) Max() int64 {
if len(this.Ranges) > 0 {
return this.Ranges[len(this.Ranges)-1][1]
@@ -135,6 +183,11 @@ func (this *PartialRanges) Max() int64 {
return 0
}
// Reset 重置范围信息
func (this *PartialRanges) Reset() {
this.Ranges = [][2]int64{}
}
func (this *PartialRanges) merge(index int) {
// forward
var lastIndex = index
@@ -187,3 +240,7 @@ func (this *PartialRanges) max(n1 int64, n2 int64) int64 {
}
return n2
}
func (this *PartialRanges) formatInt64(i int64) string {
return strconv.FormatInt(i, 10)
}

View File

@@ -3,14 +3,16 @@
package caches_test
import (
"encoding/json"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/assert"
"github.com/iwind/TeaGo/logs"
"testing"
"time"
)
func TestNewPartialRanges(t *testing.T) {
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(1, 100)
r.Add(50, 300)
@@ -28,7 +30,7 @@ func TestNewPartialRanges(t *testing.T) {
func TestNewPartialRanges1(t *testing.T) {
var a = assert.NewAssertion(t)
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(1, 100)
r.Add(1, 101)
r.Add(1, 102)
@@ -47,7 +49,7 @@ func TestNewPartialRanges1(t *testing.T) {
func TestNewPartialRanges2(t *testing.T) {
// low -> high
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(1, 100)
r.Add(1, 101)
r.Add(1, 102)
@@ -63,7 +65,7 @@ func TestNewPartialRanges2(t *testing.T) {
func TestNewPartialRanges3(t *testing.T) {
// high -> low
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(301, 302)
r.Add(303, 304)
r.Add(200, 300)
@@ -75,7 +77,7 @@ func TestNewPartialRanges3(t *testing.T) {
func TestNewPartialRanges4(t *testing.T) {
// nearby
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(301, 302)
r.Add(303, 304)
r.Add(305, 306)
@@ -90,7 +92,7 @@ func TestNewPartialRanges4(t *testing.T) {
}
func TestNewPartialRanges5(t *testing.T) {
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
}
@@ -100,7 +102,7 @@ func TestNewPartialRanges5(t *testing.T) {
func TestNewPartialRanges_Nearest(t *testing.T) {
{
// nearby
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(301, 400)
r.Add(401, 500)
r.Add(501, 600)
@@ -112,7 +114,7 @@ func TestNewPartialRanges_Nearest(t *testing.T) {
{
// nearby
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(301, 400)
r.Add(450, 500)
r.Add(550, 600)
@@ -131,45 +133,100 @@ func TestNewPartialRanges_Large_Range(t *testing.T) {
var largeSize int64 = 10000000000000
t.Log(largeSize/1024/1024/1024, "G")
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
r.Add(1, largeSize)
jsonData, err := r.AsJSON()
if err != nil {
t.Fatal(err)
}
t.Log(string(jsonData))
var s = r.String()
t.Log(s)
r2, err := caches.NewPartialRangesFromJSON(jsonData)
r2, err := caches.NewPartialRangesFromData([]byte(s))
if err != nil {
t.Fatal(err)
}
a.IsTrue(largeSize == r2.Ranges[0][1])
logs.PrintAsJSON(r, t)
}
func TestNewPartialRanges_AsJSON(t *testing.T) {
var r = caches.NewPartialRanges()
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
func TestPartialRanges_Encode_JSON(t *testing.T) {
var r = caches.NewPartialRanges(0)
for i := 0; i < 10; i++ {
r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
}
data, err := r.AsJSON()
var before = time.Now()
data, err := json.Marshal(r)
if err != nil {
t.Fatal(err)
}
t.Log(string(data))
t.Log(time.Since(before).Seconds()*1000, "ms")
t.Log(len(data))
}
r2, err := caches.NewPartialRangesFromJSON(data)
func TestPartialRanges_Encode_String(t *testing.T) {
var r = caches.NewPartialRanges(0)
r.BodySize = 1024
for i := 0; i < 10; i++ {
r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
}
var before = time.Now()
var data = r.String()
t.Log(time.Since(before).Seconds()*1000, "ms")
t.Log(len(data))
r2, err := caches.NewPartialRangesFromData([]byte(data))
if err != nil {
t.Fatal(err)
}
t.Log(r2.Ranges)
logs.PrintAsJSON(r2, t)
}
func TestPartialRanges_Version(t *testing.T) {
{
ranges, err := caches.NewPartialRangesFromData([]byte(`e:1668928495
r:0,1048576
r:1140260864,1140295164`))
if err != nil {
t.Fatal(err)
}
t.Log("version:", ranges.Version)
}
{
ranges, err := caches.NewPartialRangesFromData([]byte(`e:1668928495
r:0,1048576
r:1140260864,1140295164
v:0
`))
if err != nil {
t.Fatal(err)
}
t.Log("version:", ranges.Version)
}
{
ranges, err := caches.NewPartialRangesFromJSON([]byte(`{}`))
if err != nil {
t.Fatal(err)
}
t.Log("version:", ranges.Version)
}
}
func BenchmarkNewPartialRanges(b *testing.B) {
for i := 0; i < b.N; i++ {
var r = caches.NewPartialRanges()
var r = caches.NewPartialRanges(0)
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
}
}
}
func BenchmarkPartialRanges_String(b *testing.B) {
var r = caches.NewPartialRanges(0)
r.BodySize = 1024
for i := 0; i < 10; i++ {
r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = r.String()
}
}

View File

@@ -42,7 +42,7 @@ func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
this.header = this.openFile.header
}
isOk := false
var isOk = false
if autoDiscard {
defer func() {
@@ -67,17 +67,17 @@ func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
this.expiresAt = int64(binary.BigEndian.Uint32(buf[:SizeExpiresAt]))
status := types.Int(string(buf[OffsetStatus : OffsetStatus+SizeStatus]))
var status = types.Int(string(buf[OffsetStatus : OffsetStatus+SizeStatus]))
if status < 100 || status > 999 {
return errors.New("invalid status")
}
this.status = status
// URL
urlLength := binary.BigEndian.Uint32(buf[OffsetURLLength : OffsetURLLength+SizeURLLength])
var urlLength = binary.BigEndian.Uint32(buf[OffsetURLLength : OffsetURLLength+SizeURLLength])
// header
headerSize := int(binary.BigEndian.Uint32(buf[OffsetHeaderLength : OffsetHeaderLength+SizeHeaderLength]))
var headerSize = int(binary.BigEndian.Uint32(buf[OffsetHeaderLength : OffsetHeaderLength+SizeHeaderLength]))
if headerSize == 0 {
return nil
}
@@ -86,7 +86,7 @@ func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
// body
this.bodyOffset = this.headerOffset + int64(headerSize)
bodySize := int(binary.BigEndian.Uint64(buf[OffsetBodyLength : OffsetBodyLength+SizeBodyLength]))
var bodySize = int(binary.BigEndian.Uint64(buf[OffsetBodyLength : OffsetBodyLength+SizeBodyLength]))
if bodySize == 0 {
isOk = true
return nil
@@ -158,7 +158,7 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
return nil
}
isOk := false
var isOk = false
defer func() {
if !isOk {
@@ -171,7 +171,7 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
return err
}
headerSize := this.headerSize
var headerSize = this.headerSize
for {
n, err := this.fp.Read(buf)
@@ -215,7 +215,11 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
}
func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
isOk := false
if this.bodySize == 0 {
return nil
}
var isOk = false
defer func() {
if !isOk {
@@ -257,15 +261,22 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
}
func (this *FileReader) Read(buf []byte) (n int, err error) {
if this.bodySize == 0 {
n = 0
err = io.EOF
return
}
n, err = this.fp.Read(buf)
if err != nil && err != io.EOF {
_ = this.discard()
}
return
}
func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error {
isOk := false
var isOk = false
defer func() {
if !isOk {
@@ -273,7 +284,7 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
}
}()
offset := start
var offset = start
if start < 0 {
offset = this.bodyOffset + this.bodySize + end
end = this.bodyOffset + this.bodySize - 1
@@ -296,7 +307,7 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
for {
n, err := this.fp.Read(buf)
if n > 0 {
n2 := int(end-offset) + 1
var n2 = int(end-offset) + 1
if n2 <= n {
_, e := callback(n2)
if e != nil {
@@ -344,12 +355,12 @@ func (this *FileReader) FP() *os.File {
}
func (this *FileReader) Close() error {
if this.openFileCache != nil {
if this.isClosed {
return nil
}
this.isClosed = true
if this.isClosed {
return nil
}
this.isClosed = true
if this.openFileCache != nil {
if this.openFile != nil {
this.openFileCache.Put(this.fp.Name(), this.openFile)
} else {
@@ -359,6 +370,7 @@ func (this *FileReader) Close() error {
}
return nil
}
return this.fp.Close()
}

View File

@@ -19,7 +19,7 @@ func TestFileReader(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, path := storage.keyPath("my-key")
_, path, _ := storage.keyPath("my-key")
fp, err := os.Open(path)
if err != nil {
@@ -105,7 +105,7 @@ func TestFileReader_Range(t *testing.T) {
}
_ = writer.Close()**/
_, path := storage.keyPath("my-number")
_, path, _ := storage.keyPath("my-number")
fp, err := os.Open(path)
if err != nil {

View File

@@ -117,13 +117,10 @@ func (this *PartialFileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.
r2, ok = this.ranges.Nearest(r.Start(), r.End())
if ok && this.bodySize > 0 {
// 考虑可配置
var span int64 = 512 * 1024
if this.bodySize > 1<<30 {
span = 1 << 20
}
const minSpan = 128 << 10
// 这里限制返回的最小缓存,防止因为返回的内容过小而导致请求过多
if r2.Length() < r.Length() && r2.Length() < span {
if r2.Length() < r.Length() && r2.Length() < minSpan {
ok = false
}
}
@@ -138,6 +135,10 @@ func (this *PartialFileReader) MaxLength() int64 {
return this.ranges.Max() + 1
}
func (this *PartialFileReader) Ranges() *PartialRanges {
return this.ranges
}
func (this *PartialFileReader) discard() error {
_ = os.Remove(this.rangePath)
return this.FileReader.discard()

View File

@@ -21,6 +21,7 @@ import (
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"golang.org/x/sys/unix"
"golang.org/x/text/language"
"golang.org/x/text/message"
"math"
@@ -48,8 +49,7 @@ const (
SizeBodyLength = 8
OffsetBodyLength = OffsetHeaderLength + SizeHeaderLength
SizeMeta = SizeExpiresAt + SizeStatus + SizeURLLength + SizeHeaderLength + SizeBodyLength
OffsetKey = SizeMeta
SizeMeta = SizeExpiresAt + SizeStatus + SizeURLLength + SizeHeaderLength + SizeBodyLength
)
const (
@@ -58,6 +58,7 @@ const (
HotItemLifeSeconds int64 = 3600 // 热点数据生命周期
FileToMemoryMaxSize = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
FileTmpSuffix = ".tmp"
MinDiskSpace = 5 << 30 // 当前磁盘最小剩余空间
)
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
@@ -65,11 +66,13 @@ var sharedWritingFileKeyLocker = sync.Mutex{}
var maxOpenFiles = NewMaxOpenFiles()
const maxOpenFilesSlowCost = 500 * time.Microsecond // 0.5ms
const maxOpenFilesSlowCost = 1000 * time.Microsecond // us
const protectingLoadWhenDump = false
// FileStorage 文件缓存
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
//
// 文件结构:
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
type FileStorage struct {
policy *serverconfigs.HTTPCachePolicy
options *serverconfigs.HTTPFileCacheStorage // 二级缓存
@@ -88,6 +91,11 @@ type FileStorage struct {
ignoreKeys *setutils.FixedSet
openFileCache *OpenFileCache
mainDir string
mainDiskIsFull bool
subDirs []*FileDir
}
func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
@@ -151,6 +159,16 @@ func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy)
return
}
var subDirs = []*FileDir{}
for _, subDir := range newOptions.SubDirs {
subDirs = append(subDirs, &FileDir{
Path: subDir.Path,
Capacity: subDir.Capacity,
IsFull: false,
})
}
this.checkDiskSpace()
err = newOptions.Init()
if err != nil {
remotelogs.Error("CACHE", "update policy '"+types.String(this.policy.Id)+"' failed: init options failed: "+err.Error())
@@ -177,7 +195,7 @@ func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy)
// open cache
oldOpenFileCacheJSON, _ := json.Marshal(oldOpenFileCache)
newOpenFileCacheJSON, _ := json.Marshal(this.options.OpenFileCache)
if bytes.Compare(oldOpenFileCacheJSON, newOpenFileCacheJSON) != 0 {
if !bytes.Equal(oldOpenFileCacheJSON, newOpenFileCacheJSON) {
this.initOpenFileCache()
}
@@ -213,6 +231,19 @@ func (this *FileStorage) Init() error {
this.options.Dir = filepath.Clean(this.options.Dir)
var dir = this.options.Dir
var subDirs = []*FileDir{}
for _, subDir := range this.options.SubDirs {
subDirs = append(subDirs, &FileDir{
Path: subDir.Path,
Capacity: subDir.Capacity,
IsFull: false,
})
}
this.subDirs = subDirs
if len(subDirs) > 0 {
this.checkDiskSpace()
}
if len(dir) == 0 {
return errors.New("[CACHE]cache storage dir can not be empty")
}
@@ -285,6 +316,9 @@ func (this *FileStorage) Init() error {
// open file cache
this.initOpenFileCache()
// 检查磁盘空间
this.checkDiskSpace()
return nil
}
@@ -312,7 +346,7 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
}
}
hash, path := this.keyPath(key)
hash, path, _ := this.keyPath(key)
// 检查文件记录是否已过期
if !useStale {
@@ -380,16 +414,16 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
}
// OpenWriter 打开缓存文件等待写入
func (this *FileStorage) OpenWriter(key string, expiresAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error) {
return this.openWriter(key, expiresAt, status, size, maxSize, isPartial, false)
func (this *FileStorage) OpenWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error) {
return this.openWriter(key, expiresAt, status, headerSize, bodySize, maxSize, isPartial, false)
}
// OpenFlushWriter 打开从其他媒介直接刷入的写入器
func (this *FileStorage) OpenFlushWriter(key string, expiresAt int64, status int) (Writer, error) {
return this.openWriter(key, expiresAt, status, -1, -1, false, true)
func (this *FileStorage) OpenFlushWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64) (Writer, error) {
return this.openWriter(key, expiresAt, status, headerSize, bodySize, -1, false, true)
}
func (this *FileStorage) openWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool, isFlushing bool) (Writer, error) {
func (this *FileStorage) openWriter(key string, expiredAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool, isFlushing bool) (Writer, error) {
// 是否正在退出
if teaconst.IsQuiting {
return nil, ErrWritingUnavailable
@@ -407,8 +441,8 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
maxMemorySize = maxSize
}
var memoryStorage = this.memoryStorage
if !isFlushing && !isPartial && memoryStorage != nil && ((size > 0 && size < maxMemorySize) || size < 0) {
writer, err := memoryStorage.OpenWriter(key, expiredAt, status, size, maxMemorySize, false)
if !isFlushing && !isPartial && memoryStorage != nil && ((bodySize > 0 && bodySize < maxMemorySize) || bodySize < 0) {
writer, err := memoryStorage.OpenWriter(key, expiredAt, status, headerSize, bodySize, maxMemorySize, false)
if err == nil {
return writer, nil
}
@@ -461,17 +495,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
var hash = stringutil.Md5(key)
// TODO 可以只stat一次
var dir = this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
_, err = os.Stat(dir)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
err = os.MkdirAll(dir, 0777)
if err != nil {
return nil, err
}
dir, diskIsFull := this.subDir(hash)
if diskIsFull {
return nil, NewCapacityError("the disk is full")
}
// 检查缓存是否已经生成
@@ -518,19 +544,38 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
// 从已经存储的内容中读取信息
var isNewCreated = true
var partialBodyOffset int64
var partialRanges *PartialRanges
if isPartial {
readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
if err == nil {
var partialReader = NewPartialFileReader(readerFp)
err = partialReader.Init()
_ = partialReader.Close()
if err == nil && partialReader.bodyOffset > 0 {
isNewCreated = false
partialBodyOffset = partialReader.bodyOffset
} else {
_ = this.removeCacheFile(tmpPath)
// 数据库中是否存在
existsCacheItem, _ := this.list.Exist(hash)
if existsCacheItem {
readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
if err == nil {
var partialReader = NewPartialFileReader(readerFp)
err = partialReader.Init()
_ = partialReader.Close()
if err == nil && partialReader.bodyOffset > 0 {
partialRanges = partialReader.Ranges()
if bodySize > 0 && partialRanges != nil && partialRanges.BodySize > 0 && bodySize != partialRanges.BodySize {
_ = this.removeCacheFile(tmpPath)
} else {
isNewCreated = false
partialBodyOffset = partialReader.bodyOffset
}
} else {
_ = this.removeCacheFile(tmpPath)
}
}
}
if isNewCreated {
err = this.list.Remove(hash)
if err != nil {
return nil, err
}
}
if partialRanges == nil {
partialRanges = NewPartialRanges(expiredAt)
}
}
var flags = os.O_CREATE | os.O_WRONLY
@@ -540,7 +585,16 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
var before = time.Now()
writer, err := os.OpenFile(tmpPath, flags, 0666)
if err != nil {
return nil, err
// TODO 检查在各个系统中的稳定性
if os.IsNotExist(err) {
_ = os.MkdirAll(dir, 0777)
// open file again
writer, err = os.OpenFile(tmpPath, flags, 0666)
}
if err != nil {
return nil, err
}
}
if !isFlushing {
if time.Since(before) >= maxOpenFilesSlowCost {
@@ -572,9 +626,12 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
return nil, ErrFileIsWriting
}
var metaBodySize int64 = -1
var metaHeaderSize = -1
if isNewCreated {
// 写入过期时间
var metaBytes = make([]byte, SizeMeta+len(key))
// 写入meta
// 从v0.5.8开始不再在meta中写入Key
var metaBytes = make([]byte, SizeMeta)
binary.BigEndian.PutUint32(metaBytes[OffsetExpiresAt:], uint32(expiredAt))
// 写入状态码
@@ -583,17 +640,17 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
}
copy(metaBytes[OffsetStatus:], strconv.Itoa(status))
// 写入URL长度
binary.BigEndian.PutUint32(metaBytes[OffsetURLLength:], uint32(len(key)))
// 写入Header Length
binary.BigEndian.PutUint32(metaBytes[OffsetHeaderLength:], uint32(0))
if headerSize > 0 {
binary.BigEndian.PutUint32(metaBytes[OffsetHeaderLength:], uint32(headerSize))
metaHeaderSize = headerSize
}
// 写入Body Length
binary.BigEndian.PutUint64(metaBytes[OffsetBodyLength:], uint64(0))
// 写入URL
copy(metaBytes[OffsetKey:], key)
if bodySize > 0 {
binary.BigEndian.PutUint64(metaBytes[OffsetBodyLength:], uint64(bodySize))
metaBodySize = bodySize
}
_, err = writer.Write(metaBytes)
if err != nil {
@@ -603,12 +660,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
isOk = true
if isPartial {
ranges, err := NewPartialRangesFromFile(cachePathName + "@ranges.cache")
if err != nil {
ranges = NewPartialRanges()
}
return NewPartialFileWriter(writer, key, expiredAt, isNewCreated, isPartial, partialBodyOffset, ranges, func() {
return NewPartialFileWriter(writer, key, expiredAt, metaHeaderSize, metaBodySize, isNewCreated, isPartial, partialBodyOffset, partialRanges, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
if len(sharedWritingFileKeyMap) == 0 {
@@ -617,7 +669,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
sharedWritingFileKeyLocker.Unlock()
}), nil
} else {
return NewFileWriter(this, writer, key, expiredAt, -1, func() {
return NewFileWriter(this, writer, key, expiredAt, metaHeaderSize, metaBodySize, -1, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
if len(sharedWritingFileKeyMap) == 0 {
@@ -644,7 +696,7 @@ func (this *FileStorage) AddToList(item *Item) {
}
item.MetaSize = SizeMeta + 128
hash := stringutil.Md5(item.Key)
var hash = stringutil.Md5(item.Key)
err := this.list.Add(hash, item)
if err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
remotelogs.Error("CACHE", "add to list failed: "+err.Error())
@@ -658,15 +710,12 @@ func (this *FileStorage) Delete(key string) error {
return nil
}
this.locker.Lock()
defer this.locker.Unlock()
// 先尝试内存缓存
this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
_ = memoryStorage.Delete(key)
})
hash, path := this.keyPath(key)
hash, path, _ := this.keyPath(key)
err := this.list.Remove(hash)
if err != nil {
return err
@@ -681,9 +730,6 @@ func (this *FileStorage) Delete(key string) error {
// Stat 统计
func (this *FileStorage) Stat() (*Stat, error) {
this.locker.RLock()
defer this.locker.RUnlock()
return this.list.Stat(func(hash string) bool {
return true
})
@@ -706,57 +752,72 @@ func (this *FileStorage) CleanAll() error {
// 删除缓存和目录
// 不能直接删除子目录,比较危险
dir := this.dir()
fp, err := os.Open(dir)
if err != nil {
return err
}
defer func() {
_ = fp.Close()
}()
stat, err := fp.Stat()
if err != nil {
return err
}
if !stat.IsDir() {
return nil
}
// 改成待删除
subDirs, err := fp.Readdir(-1)
if err != nil {
return err
}
for _, info := range subDirs {
subDir := info.Name()
// 检查目录名
ok, err := regexp.MatchString(`^[0-9a-f]{2}$`, subDir)
if err != nil {
return err
}
if !ok {
continue
var rootDirs = []string{this.options.Dir}
var subDirs = this.subDirs // copy slice
if len(subDirs) > 0 {
for _, subDir := range subDirs {
rootDirs = append(rootDirs, subDir.Path)
}
}
// 修改目录名
tmpDir := dir + "/" + subDir + "-deleted"
err = os.Rename(dir+"/"+subDir, tmpDir)
var dirNameReg = regexp.MustCompile(`^[0-9a-f]{2}$`)
for _, rootDir := range rootDirs {
var dir = rootDir + "/p" + types.String(this.policy.Id)
err = func(dir string) error {
fp, err := os.Open(dir)
if err != nil {
return err
}
defer func() {
_ = fp.Close()
}()
stat, err := fp.Stat()
if err != nil {
return err
}
if !stat.IsDir() {
return nil
}
// 改成待删除
subDirs, err := fp.Readdir(-1)
if err != nil {
return err
}
for _, info := range subDirs {
subDir := info.Name()
// 检查目录名
if !dirNameReg.MatchString(subDir) {
continue
}
// 修改目录名
tmpDir := dir + "/" + subDir + "-deleted"
err = os.Rename(dir+"/"+subDir, tmpDir)
if err != nil {
return err
}
}
// 重新遍历待删除
goman.New(func() {
err = this.cleanDeletedDirs(dir)
if err != nil {
remotelogs.Warn("CACHE", "delete '*-deleted' dirs failed: "+err.Error())
}
})
return nil
}(dir)
if err != nil {
return err
}
}
// 重新遍历待删除
goman.New(func() {
err = this.cleanDeletedDirs(dir)
if err != nil {
remotelogs.Warn("CACHE", "delete '*-deleted' dirs failed: "+err.Error())
}
})
return nil
}
@@ -767,9 +828,6 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
return nil
}
this.locker.Lock()
defer this.locker.Unlock()
// 先尝试内存缓存
this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
_ = memoryStorage.Purge(keys, urlType)
@@ -778,6 +836,19 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
// 目录
if urlType == "dir" {
for _, key := range keys {
// 检查是否有通配符 http(s)://*.example.com
var schemeIndex = strings.Index(key, "://")
if schemeIndex > 0 {
var keyRight = key[schemeIndex+3:]
if strings.HasPrefix(keyRight, "*.") {
err := this.list.CleanMatchPrefix(key)
if err != nil {
return err
}
continue
}
}
err := this.list.CleanPrefix(key)
if err != nil {
return err
@@ -788,7 +859,21 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
// URL
for _, key := range keys {
hash, path := this.keyPath(key)
// 检查是否有通配符 http(s)://*.example.com
var schemeIndex = strings.Index(key, "://")
if schemeIndex > 0 {
var keyRight = key[schemeIndex+3:]
if strings.HasPrefix(keyRight, "*.") {
err := this.list.CleanMatchKey(key)
if err != nil {
return err
}
continue
}
}
// 普通的Key
hash, path, _ := this.keyPath(key)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
return err
@@ -859,25 +944,22 @@ func (this *FileStorage) CanSendfile() bool {
return this.options.EnableSendfile
}
// 绝对路径
func (this *FileStorage) dir() string {
return this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/"
}
// 获取Key对应的文件路径
func (this *FileStorage) keyPath(key string) (hash string, path string) {
func (this *FileStorage) keyPath(key string) (hash string, path string, diskIsFull bool) {
hash = stringutil.Md5(key)
dir := this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
var dir string
dir, diskIsFull = this.subDir(hash)
path = dir + "/" + hash + ".cache"
return
}
// 获取Hash对应的文件路径
func (this *FileStorage) hashPath(hash string) (path string) {
func (this *FileStorage) hashPath(hash string) (path string, diskIsFull bool) {
if len(hash) != 32 {
return ""
return "", false
}
dir := this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
var dir string
dir, diskIsFull = this.subDir(hash)
path = dir + "/" + hash + ".cache"
return
}
@@ -935,19 +1017,39 @@ func (this *FileStorage) initList() error {
}
// 清理任务
// TODO purge每个分区
func (this *FileStorage) purgeLoop() {
// 检查磁盘剩余空间
this.checkDiskSpace()
// 计算是否应该开启LFU清理
var capacityBytes = this.policy.CapacityBytes()
var capacityBytes = this.diskCapacityBytes()
var startLFU = false
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
var lfuFreePercent = this.policy.PersistenceLFUFreePercent
if lfuFreePercent <= 0 {
lfuFreePercent = 5
}
if capacityBytes > 0 {
if lfuFreePercent < 100 {
if usedPercent >= 100-lfuFreePercent {
startLFU = true
var hasFullDisk = this.mainDiskIsFull
if !hasFullDisk {
var subDirs = this.subDirs // copy slice
for _, subDir := range subDirs {
if subDir.IsFull {
hasFullDisk = true
break
}
}
}
if hasFullDisk {
startLFU = true
} else {
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
if capacityBytes > 0 {
if lfuFreePercent < 100 {
if usedPercent >= 100-lfuFreePercent {
startLFU = true
}
}
}
}
@@ -972,7 +1074,7 @@ func (this *FileStorage) purgeLoop() {
}
for i := 0; i < times; i++ {
countFound, err := this.list.Purge(purgeCount, func(hash string) error {
path := this.hashPath(hash)
path, _ := this.hashPath(hash)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
@@ -1006,7 +1108,7 @@ func (this *FileStorage) purgeLoop() {
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
err := this.list.PurgeLFU(count, func(hash string) error {
path := this.hashPath(hash)
path, _ := this.hashPath(hash)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
@@ -1087,7 +1189,7 @@ func (this *FileStorage) hotLoop() {
expiresAt = bestExpiresAt
}
writer, err := memoryStorage.openWriter(item.Key, expiresAt, reader.Status(), reader.BodySize(), -1, false)
writer, err := memoryStorage.openWriter(item.Key, expiresAt, reader.Status(), types.Int(reader.HeaderSize()), reader.BodySize(), -1, false)
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "transfer hot item failed: "+err.Error())
@@ -1111,9 +1213,12 @@ func (this *FileStorage) hotLoop() {
}
err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
_, err = writer.Write(buf[:n])
if err == nil {
goNext = true
goNext = true
if n > 0 {
_, err = writer.Write(buf[:n])
if err != nil {
goNext = false
}
}
return
})
@@ -1126,6 +1231,7 @@ func (this *FileStorage) hotLoop() {
memoryStorage.AddToList(&Item{
Type: writer.ItemType(),
Key: item.Key,
Host: ParseHost(item.Key),
ExpiredAt: expiresAt,
HeaderSize: writer.HeaderSize(),
BodySize: writer.BodySize(),
@@ -1325,3 +1431,63 @@ func (this *FileStorage) runMemoryStorageSafety(f func(memoryStorage *MemoryStor
f(memoryStorage)
}
}
// 检查磁盘剩余空间
func (this *FileStorage) checkDiskSpace() {
if this.options != nil && len(this.options.Dir) > 0 {
var stat unix.Statfs_t
err := unix.Statfs(this.options.Dir, &stat)
if err == nil {
var availableBytes = stat.Bavail * uint64(stat.Bsize)
this.mainDiskIsFull = availableBytes < MinDiskSpace
}
}
var subDirs = this.subDirs // copy slice
for _, subDir := range subDirs {
var stat unix.Statfs_t
err := unix.Statfs(subDir.Path, &stat)
if err == nil {
var availableBytes = stat.Bavail * uint64(stat.Bsize)
subDir.IsFull = availableBytes < MinDiskSpace
}
}
}
// 获取目录
func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
var suffix = "/p" + types.String(this.policy.Id) + "/" + hash[:2] + "/" + hash[2:4]
if len(hash) < 4 {
return this.options.Dir + suffix, this.mainDiskIsFull
}
var subDirs = this.subDirs // copy slice
var countSubDirs = len(subDirs)
if countSubDirs == 0 {
return this.options.Dir + suffix, this.mainDiskIsFull
}
countSubDirs++ // add main dir
// 最多只支持16个目录
if countSubDirs > 16 {
countSubDirs = 16
}
var dirIndex = this.charCode(hash[0]) % uint8(countSubDirs)
if dirIndex == 0 {
return this.options.Dir + suffix, this.mainDiskIsFull
}
var subDir = subDirs[dirIndex-1]
return subDir.Path + suffix, subDir.IsFull
}
func (this *FileStorage) charCode(r byte) uint8 {
if r >= '0' && r <= '9' {
return r - '0'
}
if r >= 'a' && r <= 'z' {
return r - 'a' + 10
}
return 0
}

View File

@@ -8,7 +8,7 @@ import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"io/ioutil"
"io"
"net/http"
"runtime"
"strconv"
@@ -62,7 +62,7 @@ func TestFileStorage_OpenWriter(t *testing.T) {
header := []byte("Header")
body := []byte("This is Body")
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, false)
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -100,7 +100,7 @@ func TestFileStorage_OpenWriter_Partial(t *testing.T) {
t.Fatal(err)
}
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, true)
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, -1, true)
if err != nil {
t.Fatal(err)
}
@@ -139,7 +139,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
t.Log(time.Since(now).Seconds()*1000, "ms")
}()
writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, -1, false)
writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -152,7 +152,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
"Last-Modified": []string{"Wed, 06 Jan 2021 10:03:29 GMT"},
"Server": []string{"CDN-Server"},
},
Body: ioutil.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
Body: io.NopCloser(bytes.NewBuffer([]byte("THIS IS HTTP BODY"))),
}
for k, v := range resp.Header {
@@ -212,7 +212,7 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
go func(i int) {
defer wg.Done()
writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, -1, false)
if err != nil {
if err != ErrFileIsWriting {
t.Error(err)
@@ -267,7 +267,7 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
go func(i int) {
defer wg.Done()
writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, -1, false)
if err != nil {
if err != ErrFileIsWriting {
t.Error(err)
@@ -522,7 +522,7 @@ func TestFileStorage_DecodeFile(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, path := storage.keyPath("my-key")
_, path, _ := storage.keyPath("my-key")
t.Log(path)
}
@@ -569,6 +569,6 @@ func BenchmarkFileStorage_KeyPath(b *testing.B) {
}
for i := 0; i < b.N; i++ {
_, _ = storage.keyPath(strconv.Itoa(i))
_, _, _ = storage.keyPath(strconv.Itoa(i))
}
}

View File

@@ -14,10 +14,10 @@ type StorageInterface interface {
// OpenWriter 打开缓存写入器等待写入
// size 和 maxSize 可能为-1
OpenWriter(key string, expiresAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error)
OpenWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error)
// OpenFlushWriter 打开从其他媒介直接刷入的写入器
OpenFlushWriter(key string, expiresAt int64, status int) (Writer, error)
OpenFlushWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64) (Writer, error)
// Delete 删除某个键值对应的缓存
Delete(key string) error

View File

@@ -1,7 +1,6 @@
package caches
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -17,6 +16,7 @@ import (
"math"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
@@ -95,7 +95,8 @@ func (this *MemoryStorage) Init() error {
// 启动定时Flush memory to disk任务
if this.parentStorage != nil {
// TODO 应该根据磁盘性能决定线程数
var threads = 1
// TODO 线程数应该可以在缓存策略和节点中设定
var threads = runtime.NumCPU()
for i := 0; i < threads; i++ {
goman.New(func() {
@@ -148,7 +149,7 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool)
}
// OpenWriter 打开缓存写入器等待写入
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error) {
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error) {
if this.ignoreKeys.Has(key) {
return nil, ErrEntityTooLarge
}
@@ -157,15 +158,15 @@ func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, s
if isPartial {
return nil, ErrFileIsWriting
}
return this.openWriter(key, expiredAt, status, size, maxSize, true)
return this.openWriter(key, expiredAt, status, headerSize, bodySize, maxSize, true)
}
// OpenFlushWriter 打开从其他媒介直接刷入的写入器
func (this *MemoryStorage) OpenFlushWriter(key string, expiresAt int64, status int) (Writer, error) {
return this.openWriter(key, expiresAt, status, -1, -1, true)
func (this *MemoryStorage) OpenFlushWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64) (Writer, error) {
return this.openWriter(key, expiresAt, status, headerSize, bodySize, -1, true)
}
func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, size int64, maxSize int64, isDirty bool) (Writer, error) {
func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64, maxSize int64, isDirty bool) (Writer, error) {
// 待写入队列是否已满
if isDirty &&
this.parentStorage != nil &&
@@ -206,10 +207,10 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, s
return nil, NewCapacityError("write memory cache failed: too many keys in cache storage")
}
capacityBytes := this.memoryCapacityBytes()
if size < 0 {
size = 0
if bodySize < 0 {
bodySize = 0
}
if capacityBytes > 0 && capacityBytes <= this.totalSize+size {
if capacityBytes > 0 && capacityBytes <= this.totalSize+bodySize {
return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
}
@@ -229,10 +230,10 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, s
// Delete 删除某个键值对应的缓存
func (this *MemoryStorage) Delete(key string) error {
hash := this.hash(key)
var hash = this.hash(key)
this.locker.Lock()
delete(this.valuesMap, hash)
_ = this.list.Remove(fmt.Sprintf("%d", hash))
_ = this.list.Remove(types.String(hash))
this.locker.Unlock()
return nil
}
@@ -262,6 +263,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
// 目录
if urlType == "dir" {
for _, key := range keys {
// 检查是否有通配符 http(s)://*.example.com
var schemeIndex = strings.Index(key, "://")
if schemeIndex > 0 {
var keyRight = key[schemeIndex+3:]
if strings.HasPrefix(keyRight, "*.") {
err := this.list.CleanMatchPrefix(key)
if err != nil {
return err
}
continue
}
}
err := this.list.CleanPrefix(key)
if err != nil {
return err
@@ -272,6 +286,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
// URL
for _, key := range keys {
// 检查是否有通配符 http(s)://*.example.com
var schemeIndex = strings.Index(key, "://")
if schemeIndex > 0 {
var keyRight = key[schemeIndex+3:]
if strings.HasPrefix(keyRight, "*.") {
err := this.list.CleanMatchKey(key)
if err != nil {
return err
}
continue
}
}
err := this.Delete(key)
if err != nil {
return err
@@ -335,7 +362,12 @@ func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePol
// AddToList 将缓存添加到列表
func (this *MemoryStorage) AddToList(item *Item) {
item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/
hash := fmt.Sprintf("%d", this.hash(item.Key))
var hash = types.String(this.hash(item.Key))
if len(item.Host) == 0 {
item.Host = ParseHost(item.Key)
}
_ = this.list.Add(hash, item)
}
@@ -432,27 +464,29 @@ func (this *MemoryStorage) startFlush() {
var statCount = 0
var writeDelayMS float64 = 0
for hash := range this.dirtyChan {
for key := range this.dirtyChan {
statCount++
if statCount == 100 {
statCount = 0
loadStat, err := load.Avg()
if err == nil && loadStat != nil {
if loadStat.Load1 > 10 {
writeDelayMS = 100
} else if loadStat.Load1 > 3 {
writeDelayMS = 50
} else if loadStat.Load1 > 2 {
writeDelayMS = 10
} else {
writeDelayMS = 0
if protectingLoadWhenDump {
loadStat, err := load.Avg()
if err == nil && loadStat != nil {
if loadStat.Load1 > 10 {
writeDelayMS = 100
} else if loadStat.Load1 > 3 {
writeDelayMS = 50
} else if loadStat.Load1 > 2 {
writeDelayMS = 10
} else {
writeDelayMS = 0
}
}
}
}
this.flushItem(hash)
this.flushItem(key)
if writeDelayMS > 0 {
time.Sleep(time.Duration(writeDelayMS) * time.Millisecond)
@@ -474,11 +508,15 @@ func (this *MemoryStorage) flushItem(key string) {
if !ok {
return
}
if !item.IsDone || item.IsExpired() {
if !item.IsDone {
remotelogs.Error("CACHE", "flush items failed: open writer failed: item has not been done")
return
}
if item.IsExpired() {
return
}
writer, err := this.parentStorage.OpenFlushWriter(key, item.ExpiresAt, item.Status)
writer, err := this.parentStorage.OpenFlushWriter(key, item.ExpiresAt, item.Status, len(item.HeaderValue), int64(len(item.BodyValue)))
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "flush items failed: open writer failed: "+err.Error())
@@ -510,6 +548,7 @@ func (this *MemoryStorage) flushItem(key string) {
this.parentStorage.AddToList(&Item{
Type: writer.ItemType(),
Key: key,
Host: ParseHost(key),
ExpiredAt: item.ExpiresAt,
HeaderSize: writer.HeaderSize(),
BodySize: writer.BodySize(),
@@ -517,8 +556,6 @@ func (this *MemoryStorage) flushItem(key string) {
// 从内存中移除
_ = this.Delete(key)
return
}
func (this *MemoryStorage) memoryCapacityBytes() int64 {
@@ -541,7 +578,7 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 {
func (this *MemoryStorage) deleteWithoutLocker(key string) error {
hash := this.hash(key)
delete(this.valuesMap, hash)
_ = this.list.Remove(fmt.Sprintf("%d", hash))
_ = this.list.Remove(types.String(hash))
return nil
}

View File

@@ -8,20 +8,28 @@ import (
"runtime"
"runtime/debug"
"strconv"
"sync"
"testing"
"time"
)
func TestMemoryStorage_OpenWriter(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
if err != nil {
t.Fatal(err)
}
_, _ = writer.WriteHeader([]byte("Header"))
_, _ = writer.Write([]byte("Hello"))
_, _ = writer.Write([]byte(", World"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log(storage.valuesMap)
{
@@ -29,6 +37,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc")
return
} else {
t.Fatal(err)
}
@@ -62,7 +71,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
}
}
writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -101,21 +110,29 @@ func TestMemoryStorage_OpenReaderLock(t *testing.T) {
}
func TestMemoryStorage_Delete(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
{
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log(len(storage.valuesMap))
}
{
writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log(len(storage.valuesMap))
}
_ = storage.Delete("abc1")
@@ -123,14 +140,18 @@ func TestMemoryStorage_Delete(t *testing.T) {
}
func TestMemoryStorage_Stat(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log(len(storage.valuesMap))
storage.AddToList(&Item{
Key: "abc",
@@ -139,11 +160,15 @@ func TestMemoryStorage_Stat(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
t.Log(len(storage.valuesMap))
storage.AddToList(&Item{
Key: "abc1",
@@ -160,14 +185,18 @@ func TestMemoryStorage_Stat(t *testing.T) {
}
func TestMemoryStorage_CleanAll(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
var expiredAt = time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
storage.AddToList(&Item{
Key: "abc",
BodySize: 5,
@@ -175,11 +204,15 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
storage.AddToList(&Item{
Key: "abc1",
BodySize: 5,
@@ -198,11 +231,15 @@ func TestMemoryStorage_Purge(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
storage.AddToList(&Item{
Key: "abc",
BodySize: 5,
@@ -210,11 +247,15 @@ func TestMemoryStorage_Purge(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
storage.AddToList(&Item{
Key: "abc1",
BodySize: 5,
@@ -230,7 +271,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
}
func TestMemoryStorage_Expire(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
MemoryAutoPurgeInterval: 5,
}, nil)
err := storage.Init()
@@ -241,11 +282,15 @@ func TestMemoryStorage_Expire(t *testing.T) {
for i := 0; i < 1000; i++ {
expiredAt := time.Now().Unix() + int64(rands.Int(0, 60))
key := "abc" + strconv.Itoa(i)
writer, err := storage.OpenWriter(key, expiredAt, 200, -1, -1, false)
writer, err := storage.OpenWriter(key, expiredAt, 200, -1, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello"))
err = writer.Close()
if err != nil {
t.Fatal(err)
}
storage.AddToList(&Item{
Key: key,
BodySize: 5,
@@ -256,7 +301,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
}
func TestMemoryStorage_Locker(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
err := storage.Init()
if err != nil {
t.Fatal(err)
@@ -304,3 +349,30 @@ func TestMemoryStorage_Stop(t *testing.T) {
t.Log(len(m))
}
func BenchmarkValuesMap(b *testing.B) {
var m = map[uint64]*MemoryItem{}
var count = 1_000_000
for i := 0; i < count; i++ {
m[uint64(i)] = &MemoryItem{
ExpiresAt: time.Now().Unix(),
}
}
b.Log(len(m))
var locker = sync.Mutex{}
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
locker.Lock()
_, ok := m[uint64(rands.Int(0, 1_000_000))]
_ = ok
locker.Unlock()
locker.Lock()
delete(m, uint64(rands.Int(2, 1000000)))
locker.Unlock()
}
})
}

30
internal/caches/utils.go Normal file
View File

@@ -0,0 +1,30 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches
import (
"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
"net"
"strings"
)
func ParseHost(key string) string {
var schemeIndex = strings.Index(key, "://")
if schemeIndex <= 0 {
return ""
}
var firstSlashIndex = strings.Index(key[schemeIndex+3:], "/")
if firstSlashIndex <= 0 {
return ""
}
var host = key[schemeIndex+3 : schemeIndex+3+firstSlashIndex]
hostPart, _, err := net.SplitHostPort(host)
if err == nil && len(hostPart) > 0 {
host = configutils.QuoteIP(hostPart)
}
return host
}

View File

@@ -0,0 +1,51 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package caches_test
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/types"
"strconv"
"testing"
)
func TestParseHost(t *testing.T) {
for _, u := range []string{
"https://goedge.cn/hello/world",
"https://goedge.cn:8080/hello/world",
"https://goedge.cn/hello/world?v=1&t=123",
"https://[::1]:1234/hello/world?v=1&t=123",
"https://[::1]/hello/world?v=1&t=123",
"https://127.0.0.1/hello/world?v=1&t=123",
"https:/hello/world?v=1&t=123",
"123456",
} {
t.Log(u, "=>", caches.ParseHost(u))
}
}
func TestUintString(t *testing.T) {
t.Log(strconv.FormatUint(xxhash.Sum64String("https://goedge.cn/"), 10))
t.Log(strconv.FormatUint(123456789, 10))
t.Log(fmt.Sprintf("%d", 1234567890123))
}
func BenchmarkUint_String(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = strconv.FormatUint(1234567890123, 10)
}
}
func BenchmarkUint_String2(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = types.String(1234567890123)
}
}
func BenchmarkUint_String3(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = fmt.Sprintf("%d", 1234567890123)
}
}

View File

@@ -11,25 +11,32 @@ import (
)
type FileWriter struct {
storage StorageInterface
rawWriter *os.File
key string
headerSize int64
bodySize int64
expiredAt int64
maxSize int64
endFunc func()
once sync.Once
storage StorageInterface
rawWriter *os.File
key string
metaHeaderSize int
headerSize int64
metaBodySize int64 // 写入前的内容长度
bodySize int64
expiredAt int64
maxSize int64
endFunc func()
once sync.Once
}
func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, expiredAt int64, maxSize int64, endFunc func()) *FileWriter {
func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, maxSize int64, endFunc func()) *FileWriter {
return &FileWriter{
storage: storage,
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
maxSize: maxSize,
endFunc: endFunc,
storage: storage,
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
maxSize: maxSize,
endFunc: endFunc,
metaHeaderSize: metaHeaderSize,
metaBodySize: metaBodySize,
}
}
@@ -45,7 +52,10 @@ func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
// WriteHeaderLength 写入Header长度数据
func (this *FileWriter) WriteHeaderLength(headerLength int) error {
bytes4 := make([]byte, 4)
if this.metaHeaderSize > 0 && this.metaHeaderSize == headerLength {
return nil
}
var bytes4 = make([]byte, 4)
binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength, io.SeekStart)
if err != nil {
@@ -88,7 +98,10 @@ func (this *FileWriter) WriteAt(offset int64, data []byte) error {
// WriteBodyLength 写入Body长度数据
func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
bytes8 := make([]byte, 8)
if this.metaBodySize >= 0 && bodyLength == this.metaBodySize {
return nil
}
var bytes8 = make([]byte, 8)
binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength, io.SeekStart)
if err != nil {
@@ -109,7 +122,7 @@ func (this *FileWriter) Close() error {
this.endFunc()
})
path := this.rawWriter.Name()
var path = this.rawWriter.Name()
err := this.WriteHeaderLength(types.Int(this.headerSize))
if err != nil {

View File

@@ -11,13 +11,18 @@ import (
)
type PartialFileWriter struct {
rawWriter *os.File
key string
headerSize int64
bodySize int64
expiredAt int64
endFunc func()
once sync.Once
rawWriter *os.File
key string
metaHeaderSize int
headerSize int64
metaBodySize int64
bodySize int64
expiredAt int64
endFunc func()
once sync.Once
isNew bool
isPartial bool
@@ -27,17 +32,19 @@ type PartialFileWriter struct {
rangePath string
}
func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
return &PartialFileWriter{
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
endFunc: endFunc,
isNew: isNew,
isPartial: isPartial,
bodyOffset: bodyOffset,
ranges: ranges,
rangePath: partialRangesFilePath(rawWriter.Name()),
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
endFunc: endFunc,
isNew: isNew,
isPartial: isPartial,
bodyOffset: bodyOffset,
ranges: ranges,
rangePath: partialRangesFilePath(rawWriter.Name()),
metaHeaderSize: metaHeaderSize,
metaBodySize: metaBodySize,
}
}
@@ -71,7 +78,11 @@ func (this *PartialFileWriter) AppendHeader(data []byte) error {
// WriteHeaderLength 写入Header长度数据
func (this *PartialFileWriter) WriteHeaderLength(headerLength int) error {
bytes4 := make([]byte, 4)
if this.metaHeaderSize > 0 && this.metaHeaderSize == headerLength {
return nil
}
var bytes4 = make([]byte, 4)
binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength, io.SeekStart)
if err != nil {
@@ -110,8 +121,13 @@ func (this *PartialFileWriter) WriteAt(offset int64, data []byte) error {
}
if this.bodyOffset == 0 {
this.bodyOffset = SizeMeta + int64(len(this.key)) + this.headerSize
var keyLength = 0
if this.ranges.Version == 0 { // 以往的版本包含有Key
keyLength = len(this.key)
}
this.bodyOffset = SizeMeta + int64(keyLength) + this.headerSize
}
_, err := this.rawWriter.WriteAt(data, this.bodyOffset+offset)
if err != nil {
return err
@@ -129,7 +145,10 @@ func (this *PartialFileWriter) SetBodyLength(bodyLength int64) {
// WriteBodyLength 写入Body长度数据
func (this *PartialFileWriter) WriteBodyLength(bodyLength int64) error {
bytes8 := make([]byte, 8)
if this.metaBodySize > 0 && this.metaBodySize == bodyLength {
return nil
}
var bytes8 = make([]byte, 8)
binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength, io.SeekStart)
if err != nil {
@@ -150,8 +169,11 @@ func (this *PartialFileWriter) Close() error {
this.endFunc()
})
this.ranges.BodySize = this.bodySize
err := this.ranges.WriteToFile(this.rangePath)
if err != nil {
_ = this.rawWriter.Close()
this.remove()
return err
}

View File

@@ -5,7 +5,6 @@ package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"testing"
"time"
@@ -16,7 +15,7 @@ func TestPartialFileWriter_Write(t *testing.T) {
_ = os.Remove(path)
var reader = func() {
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
@@ -27,8 +26,8 @@ func TestPartialFileWriter_Write(t *testing.T) {
if err != nil {
t.Fatal(err)
}
var ranges = caches.NewPartialRanges()
var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, true, true, 0, ranges, func() {
var ranges = caches.NewPartialRanges(0)
var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, -1, -1, true, true, 0, ranges, func() {
t.Log("end")
})
_, err = writer.WriteHeader([]byte("header"))

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedBrotliReaderPool *ReaderPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedDeflateReaderPool *ReaderPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedGzipReaderPool *ReaderPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -10,6 +11,10 @@ import (
var sharedZSTDReaderPool *ReaderPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -5,10 +5,46 @@ package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
stringutil "github.com/iwind/TeaGo/utils/string"
"strings"
"testing"
"time"
)
func TestBrotliWriter_LargeFile(t *testing.T) {
var data = []byte{}
for i := 0; i < 1024*1024; i++ {
data = append(data, stringutil.Rand(32)...)
}
t.Log(len(data)/1024/1024, "M")
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
t.Fatal(err)
}
var offset = 0
var size = 4096
for offset < len(data) {
_, err = writer.Write(data[offset : offset+size])
if err != nil {
t.Fatal(err)
}
offset += size
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
}
func BenchmarkBrotliWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/andybalholm/brotli"
"io"
@@ -11,6 +12,10 @@ import (
var sharedBrotliWriterPool *WriterPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -4,6 +4,7 @@ package compressions
import (
"compress/flate"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -11,6 +12,10 @@ import (
var sharedDeflateWriterPool *WriterPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -4,6 +4,7 @@ package compressions
import (
"compress/gzip"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
@@ -11,6 +12,10 @@ import (
var sharedGzipWriterPool *WriterPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,6 +3,7 @@
package compressions
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/klauspost/compress/zstd"
"io"
@@ -11,6 +12,10 @@ import (
var sharedZSTDWriterPool *WriterPool
func init() {
if !teaconst.IsMain {
return
}
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256

View File

@@ -3,20 +3,25 @@ package configs
import (
"github.com/iwind/TeaGo/Tea"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
)
// APIConfig 节点API配置
type APIConfig struct {
RPC struct {
Endpoints []string `yaml:"endpoints"`
} `yaml:"rpc"`
NodeId string `yaml:"nodeId"`
Secret string `yaml:"secret"`
Endpoints []string `yaml:"endpoints" json:"endpoints"`
DisableUpdate bool `yaml:"disableUpdate" json:"disableUpdate"`
} `yaml:"rpc" json:"rpc"`
NodeId string `yaml:"nodeId" json:"nodeId"`
Secret string `yaml:"secret" json:"secret"`
}
func NewAPIConfig() *APIConfig {
return &APIConfig{}
}
func LoadAPIConfig() (*APIConfig, error) {
data, err := ioutil.ReadFile(Tea.ConfigFile("api.yaml"))
data, err := os.ReadFile(Tea.ConfigFile("api.yaml"))
if err != nil {
return nil, err
}
@@ -30,12 +35,12 @@ func LoadAPIConfig() (*APIConfig, error) {
return config, nil
}
// 保存到文件
// WriteFile 保存到文件
func (this *APIConfig) WriteFile(path string) error {
data, err := yaml.Marshal(this)
if err != nil {
return err
}
err = ioutil.WriteFile(path, data, 0666)
err = os.WriteFile(path, data, 0666)
return err
}

View File

@@ -1,11 +1,15 @@
package configs
package configs_test
import "testing"
import (
"github.com/TeaOSLab/EdgeNode/internal/configs"
_ "github.com/iwind/TeaGo/bootstrap"
"testing"
)
func TestLoadAPIConfig(t *testing.T) {
config, err := LoadAPIConfig()
config, err := configs.LoadAPIConfig()
if err != nil {
t.Fatal(err)
}
t.Log(config)
t.Logf("%+v", config)
}

View File

@@ -1,10 +1,11 @@
package configs
// 集群配置
// ClusterConfig 集群配置
type ClusterConfig struct {
RPC struct {
Endpoints []string `yaml:"endpoints"`
} `yaml:"rpc"`
ClusterId string `yaml:"clusterId"`
Secret string `yaml:"secret"`
Endpoints []string `yaml:"endpoints" json:"endpoints"`
DisableUpdate bool `yaml:"disableUpdate" json:"disableUpdate"`
} `yaml:"rpc" json:"rpc"`
ClusterId string `yaml:"clusterId" json:"clusterId"`
Secret string `yaml:"secret" json:"secret"`
}

View File

@@ -1,5 +0,0 @@
package configs
import "sync"
var sharedLocker = &sync.RWMutex{}

7
internal/conns/linger.go Normal file
View File

@@ -0,0 +1,7 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package conns
type LingerConn interface {
SetLinger(sec int) error
}

123
internal/conns/map.go Normal file
View File

@@ -0,0 +1,123 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package conns
import (
"net"
"sync"
)
var SharedMap = NewMap()
type Map struct {
m map[string]map[int]net.Conn // ip => { port => Conn }
locker sync.RWMutex
}
func NewMap() *Map {
return &Map{
m: map[string]map[int]net.Conn{},
}
}
func (this *Map) Add(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
this.m[ip] = map[int]net.Conn{port: conn}
} else {
connMap[port] = conn
}
}
func (this *Map) Remove(conn net.Conn) {
if conn == nil {
return
}
tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
if !ok {
return
}
var ip = tcpAddr.IP.String()
var port = tcpAddr.Port
this.locker.Lock()
defer this.locker.Unlock()
connMap, ok := this.m[ip]
if !ok {
return
}
delete(connMap, port)
if len(connMap) == 0 {
delete(this.m, ip)
}
}
func (this *Map) CountIPConns(ip string) int {
this.locker.RLock()
var l = len(this.m[ip])
this.locker.RUnlock()
return l
}
func (this *Map) CloseIPConns(ip string) {
var conns = []net.Conn{}
this.locker.RLock()
connMap, ok := this.m[ip]
// 复制防止在Close时产生并发冲突
if ok {
for _, conn := range connMap {
conns = append(conns, conn)
}
}
// 需要在Close之前结束防止死循环
this.locker.RUnlock()
if ok {
for _, conn := range conns {
// 设置Linger
lingerConn, isLingerConn := conn.(LingerConn)
if isLingerConn {
_ = lingerConn.SetLinger(0)
}
// 关闭
_ = conn.Close()
}
// 这里不需要从 m 中删除,因为关闭时会自然触发回调
}
}
func (this *Map) AllConns() []net.Conn {
this.locker.RLock()
defer this.locker.RUnlock()
var result = []net.Conn{}
for _, m := range this.m {
for _, connInfo := range m {
result = append(result, connInfo)
}
}
return result
}

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.4.9"
Version = "0.6.4"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -2,7 +2,10 @@
package teaconst
import "github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"os"
)
var (
// 流量统计
@@ -12,6 +15,7 @@ var (
NodeId int64 = 0
NodeIdString = ""
IsMain = len(os.Args) == 1 || (len(os.Args) >= 2 && os.Args[1] == "pprof")
GlobalProductName = nodeconfigs.DefaultProductName

View File

@@ -1,6 +1,5 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package firewalls
@@ -10,10 +9,12 @@ import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/types"
@@ -21,11 +22,16 @@ import (
"net"
"os/exec"
"strings"
"time"
)
var SharedDDoSProtectionManager = NewDDoSProtectionManager()
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventReload, func() {
if nftablesInstance == nil {
return
@@ -53,19 +59,13 @@ func init() {
// DDoSProtectionManager DDoS防护
type DDoSProtectionManager struct {
nftPath string
lastAllowIPList []string
lastConfig []byte
}
// NewDDoSProtectionManager 获取新对象
func NewDDoSProtectionManager() *DDoSProtectionManager {
nftPath, _ := exec.LookPath("nft")
return &DDoSProtectionManager{
nftPath: nftPath,
}
return &DDoSProtectionManager{}
}
// Apply 应用配置
@@ -75,7 +75,7 @@ func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) e
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil {
var allowIPList = nodeConfig.AllowedIPs
if !utils.ContainsSameStrings(allowIPList, this.lastAllowIPList) {
if !utils.EqualStrings(allowIPList, this.lastAllowIPList) {
allowIPListChanged = true
this.lastAllowIPList = allowIPList
}
@@ -91,11 +91,14 @@ func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) e
}
remotelogs.Println("FIREWALL", "change DDoS protection config")
if len(this.nftPath) == 0 {
if len(this.nftExe()) == 0 {
return errors.New("can not find nft command")
}
if nftablesInstance == nil {
if config == nil || !config.IsOn() {
return nil
}
return errors.New("nftables instance should not be nil")
}
@@ -154,6 +157,11 @@ func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) e
// 添加TCP规则
func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig) error {
var nftExe = this.nftExe()
if len(nftExe) == 0 {
return nil
}
// 检查nft版本不能小于0.9
if len(nftablesInstance.version) > 0 && stringutil.VersionCompare("0.9", nftablesInstance.version) > 0 {
return nil
@@ -195,14 +203,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
}
}
// new connections rate
var newConnectionsRate = tcpConfig.NewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
if newConnectionsRate <= 0 {
newConnectionsRate = 100000
// new connections rate (minutely)
var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
if newConnectionsMinutelyRate <= 0 {
newConnectionsMinutelyRate = 100000
}
}
var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
if newConnectionsMinutelyRateBlockTimeout < 0 {
newConnectionsMinutelyRateBlockTimeout = 0
}
// new connections rate (secondly)
var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
if newConnectionsSecondlyRate <= 0 {
newConnectionsSecondlyRate = 10000
}
}
var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
if newConnectionsSecondlyRateBlockTimeout < 0 {
newConnectionsSecondlyRateBlockTimeout = 0
}
// 检查是否有变化
var hasChanges = false
@@ -215,7 +240,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
hasChanges = true
break
}
if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
hasChanges = true
break
}
@@ -242,33 +271,61 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
// 添加新规则
for _, port := range ports {
if maxConnections > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "count", "over", types.String(maxConnections), "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnections", types.String(maxConnections)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "count", "over", types.String(maxConnections), "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnections", types.String(maxConnections)}))
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
}
}
// TODO 让用户选择是drop还是reject
if maxConnectionsPerIP > 0 {
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
}
}
if newConnectionsRate > 0 {
// TODO 思考是否有惩罚机制
var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
var stderr = &bytes.Buffer{}
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
// 超过一定速率就drop或者加入黑名单分钟
// TODO 让用户选择是drop还是reject
if newConnectionsMinutelyRate > 0 {
if newConnectionsMinutelyRateBlockTimeout > 0 {
var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
}
} else {
var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
}
}
}
// 超过一定速率就drop或者加入黑名单
// TODO 让用户选择是drop还是reject
if newConnectionsSecondlyRate > 0 {
if newConnectionsSecondlyRateBlockTimeout > 0 {
var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
}
} else {
var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
}
}
}
}
@@ -335,14 +392,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
for _, rule := range rules {
var pieces = this.decodeUserData(rule.UserData())
if len(pieces) != 4 {
if len(pieces) < 4 {
continue
}
if pieces[0] != "tcp" {
continue
}
switch pieces[2] {
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
err := chain.DeleteRule(rule)
if err != nil {
return err
@@ -500,3 +557,8 @@ func (this *DDoSProtectionManager) updateAllowIPList(allIPList []string) error {
return nil
}
func (this *DDoSProtectionManager) nftExe() string {
path, _ := exec.LookPath("nft")
return path
}

View File

@@ -11,7 +11,6 @@ import (
var SharedDDoSProtectionManager = NewDDoSProtectionManager()
type DDoSProtectionManager struct {
nftPath string
}
func NewDDoSProtectionManager() *DDoSProtectionManager {

View File

@@ -3,6 +3,7 @@
package firewalls
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"runtime"
@@ -14,6 +15,10 @@ var firewallLocker = &sync.Mutex{}
// 初始化
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventLoaded, func() {
var firewall = Firewall()
if firewall.Name() != "mock" {

View File

@@ -0,0 +1,47 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package firewalls
import (
"github.com/iwind/TeaGo/types"
"strings"
"sync"
"time"
)
type BaseFirewall struct {
locker sync.Mutex
latestIPTimes []string // [ip@time, ....]
}
// 检查是否在最近添加过
func (this *BaseFirewall) checkLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -3,27 +3,38 @@
package firewalls
import (
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"github.com/iwind/TeaGo/types"
"os/exec"
"strings"
"time"
)
type firewalldCmd struct {
cmd *executils.Cmd
denyIP string
}
type Firewalld struct {
BaseFirewall
isReady bool
exe string
cmdQueue chan *exec.Cmd
cmdQueue chan *firewalldCmd
}
func NewFirewalld() *Firewalld {
var firewalld = &Firewalld{
cmdQueue: make(chan *exec.Cmd, 4096),
cmdQueue: make(chan *firewalldCmd, 4096),
}
path, err := exec.LookPath("firewall-cmd")
if err == nil && len(path) > 0 {
var cmd = exec.Command(path, "--state")
var cmd = executils.NewTimeoutCmd(3*time.Second, path, "--state")
err := cmd.Run()
if err == nil {
firewalld.exe = path
@@ -40,13 +51,19 @@ func NewFirewalld() *Firewalld {
func (this *Firewalld) init() {
goman.New(func() {
for cmd := range this.cmdQueue {
for c := range this.cmdQueue {
var cmd = c.cmd
err := cmd.Run()
if err != nil {
if strings.HasPrefix(err.Error(), "Warning:") {
continue
}
remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
} else {
// 关闭连接
if len(c.denyIP) > 0 {
conns.SharedMap.CloseIPConns(c.denyIP)
}
}
}
})
@@ -70,8 +87,26 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
if !this.isReady {
return nil
}
var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--add-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd, "")
return nil
}
func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol string) error {
for _, portRange := range portRanges {
var port = this.PortRangeString(portRange, protocol)
{
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--add-port="+port, "--permanent")
this.pushCmd(cmd, "")
}
{
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--add-port="+port)
this.pushCmd(cmd, "")
}
}
return nil
}
@@ -79,15 +114,45 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
if !this.isReady {
return nil
}
var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd)
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--remove-port="+types.String(port)+"/"+protocol)
this.pushCmd(cmd, "")
return nil
}
func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol string) error {
var port = this.PortRangeString(portRange, protocol)
{
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--remove-port="+port, "--permanent")
this.pushCmd(cmd, "")
}
{
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--remove-port="+port)
this.pushCmd(cmd, "")
}
return nil
}
func (this *Firewalld) PortRangeString(portRange [2]int, protocol string) string {
if portRange[0] == portRange[1] {
return types.String(portRange[0]) + "/" + protocol
} else {
return types.String(portRange[0]) + "-" + types.String(portRange[1]) + "/" + protocol
}
}
func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -96,15 +161,21 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
if timeoutSeconds > 0 {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, args...)
this.pushCmd(cmd, ip)
return nil
}
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
if !this.isReady {
return nil
}
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
@@ -113,8 +184,19 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int) error {
if timeoutSeconds > 0 {
args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, args...)
if async {
this.pushCmd(cmd, ip)
return nil
}
// 关闭连接
defer conns.SharedMap.CloseIPConns(ip)
err := cmd.Run()
if err != nil {
return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
}
return nil
}
@@ -122,21 +204,22 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
if !this.isReady {
return nil
}
var family = "ipv4"
if strings.Contains(ip, ":") {
family = "ipv6"
}
for _, action := range []string{"reject", "drop"} {
var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
var cmd = exec.Command(this.exe, args...)
this.pushCmd(cmd)
var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, args...)
this.pushCmd(cmd, "")
}
return nil
}
func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
func (this *Firewalld) pushCmd(cmd *executils.Cmd, denyIP string) {
select {
case this.cmdQueue <- cmd:
case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
default:
// we discard the command
}

View File

@@ -23,7 +23,10 @@ type FirewallInterface interface {
RejectSourceIP(ip string, timeoutSeconds int) error
// DropSourceIP 丢弃某个源IP数据
DropSourceIP(ip string, timeoutSeconds int) error
// ip 要封禁的IP
// timeoutSeconds 过期时间
// async 是否异步
DropSourceIP(ip string, timeoutSeconds int, async bool) error
// RemoveSourceIP 删除某个源IP
RemoveSourceIP(ip string) error

View File

@@ -47,7 +47,7 @@ func (this *MockFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
}
// DropSourceIP 丢弃某个源IP数据
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *MockFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
_ = ip
_ = timeoutSeconds
return nil

View File

@@ -5,11 +5,14 @@
package firewalls
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/conns"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"github.com/iwind/TeaGo/types"
"net"
"os/exec"
@@ -21,9 +24,13 @@ import (
// check nft status, if being enabled we load it automatically
func init() {
if !teaconst.IsMain {
return
}
if runtime.GOOS == "linux" {
var ticker = time.NewTicker(3 * time.Minute)
go func() {
goman.New(func() {
for range ticker.C {
// if already ready, we break
if nftablesIsReady {
@@ -48,7 +55,7 @@ func init() {
break
}
}
}()
})
}
}
@@ -74,9 +81,16 @@ func (this *nftablesTableDefinition) protocol() string {
return "ip"
}
type blockIPItem struct {
action string
ip string
timeoutSeconds int
}
func NewNFTablesFirewall() (*NFTablesFirewall, error) {
var firewall = &NFTablesFirewall{
conn: nftables.NewConn(),
conn: nftables.NewConn(),
dropIPQueue: make(chan *blockIPItem, 4096),
}
err := firewall.init()
if err != nil {
@@ -87,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
}
type NFTablesFirewall struct {
BaseFirewall
conn *nftables.Conn
isReady bool
version string
@@ -98,6 +114,8 @@ type NFTablesFirewall struct {
denyIPv6Set *nftables.Set
firewalld *Firewalld
dropIPQueue chan *blockIPItem
}
func (this *NFTablesFirewall) init() error {
@@ -243,6 +261,18 @@ func (this *NFTablesFirewall) init() error {
nftablesIsReady = true
nftablesInstance = this
goman.New(func() {
for ipItem := range this.dropIPQueue {
switch ipItem.action {
case "drop":
err = this.DropSourceIP(ipItem.ip, ipItem.timeoutSeconds, false)
if err != nil {
remotelogs.Warn("NFTABLES", "drop ip '"+ipItem.ip+"' failed: "+err.Error())
}
}
}
})
// load firewalld
var firewalld = NewFirewalld()
if firewalld.IsReady() {
@@ -307,16 +337,40 @@ func (this *NFTablesFirewall) AllowSourceIP(ip string) error {
// RejectSourceIP 拒绝某个源IP连接
// we did not create set for drop ip, so we reuse DropSourceIP() method here
func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) error {
return this.DropSourceIP(ip, timeoutSeconds)
return this.DropSourceIP(ip, timeoutSeconds, true)
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
var data = net.ParseIP(ip)
if data == nil {
return errors.New("invalid ip '" + ip + "'")
}
// 尝试关闭连接
conns.SharedMap.CloseIPConns(ip)
// 避免短时间内重复添加
if async && this.checkLatestIP(ip) {
return nil
}
if async {
select {
case this.dropIPQueue <- &blockIPItem{
action: "drop",
ip: ip,
timeoutSeconds: timeoutSeconds,
}:
default:
return errors.New("drop ip queue is full")
}
return nil
}
// 再次尝试关闭连接
defer conns.SharedMap.CloseIPConns(ip)
if strings.Contains(ip, ":") { // ipv6
if this.denyIPv6Set == nil {
return errors.New("ipv6 ip set is nil")
@@ -378,18 +432,49 @@ func (this *NFTablesFirewall) RemoveSourceIP(ip string) error {
// 读取版本号
func (this *NFTablesFirewall) readVersion(nftPath string) string {
var cmd = exec.Command(nftPath, "--version")
var output = &bytes.Buffer{}
cmd.Stdout = output
var cmd = executils.NewTimeoutCmd(10*time.Second, nftPath, "--version")
cmd.WithStdout()
err := cmd.Run()
if err != nil {
return ""
}
var outputString = output.String()
var outputString = cmd.Stdout()
var versionMatches = regexp.MustCompile(`nftables v([\d.]+)`).FindStringSubmatch(outputString)
if len(versionMatches) <= 1 {
return ""
}
return versionMatches[1]
}
// 检查是否在最近添加过
func (this *NFTablesFirewall) existLatestIP(ip string) bool {
this.locker.Lock()
defer this.locker.Unlock()
var expiredIndex = -1
for index, ipTime := range this.latestIPTimes {
var pieces = strings.Split(ipTime, "@")
var oldIP = pieces[0]
var oldTimestamp = pieces[1]
if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
expiredIndex = index
continue
}
if oldIP == ip {
return true
}
}
if expiredIndex > -1 {
this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
}
this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
const maxLen = 128
if len(this.latestIPTimes) > maxLen {
this.latestIPTimes = this.latestIPTimes[1:]
}
return false
}

View File

@@ -51,7 +51,7 @@ func (this *NFTablesFirewall) RejectSourceIP(ip string, timeoutSeconds int) erro
}
// DropSourceIP 丢弃某个源IP数据
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int) error {
func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async bool) error {
return nil
}

View File

@@ -1,6 +1,5 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
// +build linux
package nftables

View File

@@ -0,0 +1,113 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
package nftables
import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"github.com/iwind/TeaGo/logs"
"os"
"os/exec"
"runtime"
"time"
)
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventReload, func() {
// linux only
if runtime.GOOS != "linux" {
return
}
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return
}
if nodeConfig == nil || !nodeConfig.AutoInstallNftables {
return
}
if os.Getgid() == 0 { // root user only
_, err := exec.LookPath("nft")
if err == nil {
return
}
goman.New(func() {
err := NewInstaller().Install()
if err != nil {
// 不需要传到API节点
logs.Println("[NFTABLES]install nftables failed: " + err.Error())
}
})
}
})
}
type Installer struct {
}
func NewInstaller() *Installer {
return &Installer{}
}
func (this *Installer) Install() error {
// linux only
if runtime.GOOS != "linux" {
return nil
}
// 检查是否已经存在
_, err := exec.LookPath("nft")
if err == nil {
return nil
}
var cmd *executils.Cmd
// check dnf
dnfExe, err := exec.LookPath("dnf")
if err == nil {
cmd = executils.NewCmd(dnfExe, "-y", "install", "nftables")
}
// check apt
if cmd == nil {
aptExe, err := exec.LookPath("apt")
if err == nil {
cmd = executils.NewCmd(aptExe, "install", "nftables")
}
}
// check yum
if cmd == nil {
yumExe, err := exec.LookPath("yum")
if err == nil {
cmd = executils.NewCmd(yumExe, "-y", "install", "nftables")
}
}
if cmd == nil {
return nil
}
cmd.WithTimeout(10 * time.Minute)
cmd.WithStderr()
err = cmd.Run()
if err != nil {
return errors.New(err.Error() + ": " + cmd.Stderr())
}
remotelogs.Println("NFTABLES", "installed nftables with command '"+cmd.String()+"' successfully")
return nil
}

View File

@@ -1,4 +1,5 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
package nftables

View File

@@ -1,4 +1,5 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
//go:build linux
package nftables_test

View File

@@ -3,6 +3,7 @@
package goman
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"runtime"
"sync"
"time"
@@ -14,6 +15,10 @@ var instanceId = uint64(0)
// New 新创建goroutine
func New(f func()) {
if !teaconst.IsMain {
return
}
_, file, line, _ := runtime.Caller(1)
go func() {
@@ -42,6 +47,10 @@ func New(f func()) {
// NewWithArgs 创建带有参数的goroutine
func NewWithArgs(f func(args ...interface{}), args ...interface{}) {
if !teaconst.IsMain {
return
}
_, file, line, _ := runtime.Caller(1)
go func() {

View File

@@ -1,11 +1,11 @@
package iplibrary
import (
"bytes"
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"os/exec"
"runtime"
"time"
@@ -13,9 +13,9 @@ import (
// FirewalldAction Firewalld动作管理
// 常用命令:
// - 查询列表: firewall-cmd --list-all
// - 添加IPfirewall-cmd --add-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
// - 删除IPfirewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
// - 查询列表: firewall-cmd --list-all
// - 添加IPfirewall-cmd --add-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
// - 删除IPfirewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
type FirewalldAction struct {
BaseAction
@@ -144,12 +144,11 @@ func (this *FirewalldAction) runActionSingleIP(action string, listType IPListTyp
// MAC OS直接返回
return nil
}
cmd := exec.Command(path, args...)
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
cmd := executils.NewTimeoutCmd(30*time.Second, path, args...)
cmd.WithStderr()
err = cmd.Run()
if err != nil {
return errors.New(err.Error() + ", output: " + string(stderr.Bytes()))
return errors.New(err.Error() + ", output: " + cmd.Stderr())
}
return nil
}

View File

@@ -1,10 +1,10 @@
package iplibrary
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"github.com/iwind/TeaGo/types"
"os/exec"
"runtime"
@@ -16,12 +16,12 @@ import (
// IPSetAction IPSet动作
// 相关命令:
// - 利用Firewalld管理set
// - 添加firewall-cmd --permanent --new-ipset=edge_ip_list --type=hash:ip --option="timeout=0"
// - 删除firewall-cmd --permanent --delete-ipset=edge_ip_list
// - 重载firewall-cmd --reload
// - firewalld+ipset: firewall-cmd --permanent --add-rich-rule="rule source ipset='edge_ip_list' reject"
// - 添加firewall-cmd --permanent --new-ipset=edge_ip_list --type=hash:ip --option="timeout=0"
// - 删除firewall-cmd --permanent --delete-ipset=edge_ip_list
// - 重载firewall-cmd --reload
// - firewalld+ipset: firewall-cmd --permanent --add-rich-rule="rule source ipset='edge_ip_list' reject"
// - 利用IPTables管理set
// - 添加iptables -A INPUT -m set --match-set edge_ip_list src -j REJECT
// - 添加iptables -A INPUT -m set --match-set edge_ip_list src -j REJECT
// - 添加Itemipset add edge_ip_list 192.168.2.32 timeout 30
// - 删除Item: ipset del edge_ip_list 192.168.2.32
// - 创建setipset create edge_ip_list hash:ip timeout 0
@@ -30,16 +30,13 @@ import (
type IPSetAction struct {
BaseAction
config *firewallconfigs.FirewallActionIPSetConfig
errorBuf *bytes.Buffer
config *firewallconfigs.FirewallActionIPSetConfig
ipsetNotfound bool
}
func NewIPSetAction() *IPSetAction {
return &IPSetAction{
errorBuf: &bytes.Buffer{},
}
return &IPSetAction{}
}
func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) error {
@@ -68,14 +65,13 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "create", listName, "hash:ip", "timeout", "0", "maxelem", "1000000")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "create", listName, "hash:ip", "timeout", "0", "maxelem", "1000000")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
if !bytes.Contains(output, []byte("already exists")) {
return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
var output = cmd.Stderr()
if !strings.Contains(output, "already exists") {
return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + output)
} else {
err = nil
}
@@ -87,14 +83,13 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "create", listName, "hash:ip", "family", "inet6", "timeout", "0", "maxelem", "1000000")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "create", listName, "hash:ip", "family", "inet6", "timeout", "0", "maxelem", "1000000")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
if !bytes.Contains(output, []byte("already exists")) {
return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
var output = cmd.Stderr()
if !strings.Contains(output, "already exists") {
return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + output)
} else {
err = nil
}
@@ -114,16 +109,15 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if len(listName) == 0 {
continue
}
cmd := exec.Command(path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
if bytes.Contains(output, []byte("NAME_CONFLICT")) {
var output = cmd.Stderr()
if strings.Contains(output, "NAME_CONFLICT") {
err = nil
} else {
return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + output)
}
}
}
@@ -133,16 +127,15 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if len(listName) == 0 {
continue
}
cmd := exec.Command(path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=family=inet6", "--option=timeout=0", "--option=maxelem=1000000")
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=family=inet6", "--option=timeout=0", "--option=maxelem=1000000")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
if bytes.Contains(output, []byte("NAME_CONFLICT")) {
var output = cmd.Stderr()
if strings.Contains(output, "NAME_CONFLICT") {
err = nil
} else {
return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + output)
}
}
}
@@ -152,13 +145,11 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' accept")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' accept")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + cmd.Stderr())
}
}
@@ -167,25 +158,21 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' reject")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' reject")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + cmd.Stderr())
}
}
// reload
{
cmd := exec.Command(path, "--reload")
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--reload")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
return errors.New("firewall-cmd reload: " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd reload: " + err.Error() + ", output: " + cmd.Stderr())
}
}
}
@@ -204,19 +191,17 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
}
// 检查规则是否存在
var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
err := cmd.Run()
var exists = err == nil
// 添加规则
if !exists {
var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
return errors.New("iptables add rule: " + err.Error() + ", output: " + string(output))
return errors.New("iptables add rule: " + err.Error() + ", output: " + cmd.Stderr())
}
}
}
@@ -228,18 +213,16 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
}
// 检查规则是否存在
var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
err := cmd.Run()
var exists = err == nil
if !exists {
var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
cmd.WithStderr()
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
return errors.New("iptables add rule: " + err.Error() + ", output: " + string(output))
return errors.New("iptables add rule: " + err.Error() + ", output: " + cmd.Stderr())
}
}
}
@@ -361,12 +344,11 @@ func (this *IPSetAction) runActionSingleIP(action string, listType IPListType, i
return nil
}
this.errorBuf.Reset()
var cmd = exec.Command(path, args...)
cmd.Stderr = this.errorBuf
var cmd = executils.NewTimeoutCmd(30*time.Second, path, args...)
cmd.WithStderr()
err = cmd.Run()
if err != nil {
var errString = this.errorBuf.String()
var errString = cmd.Stderr()
if action == "deleteItem" && strings.Contains(errString, "not added") {
return nil
}

View File

@@ -1,20 +1,24 @@
package iplibrary
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"os/exec"
"runtime"
"strings"
"time"
)
// IPTablesAction IPTables动作
// 相关命令:
// iptables -A INPUT -s "192.168.2.32" -j ACCEPT
// iptables -A INPUT -s "192.168.2.32" -j REJECT
// iptables -D INPUT ...
// iptables -F INPUT
//
// iptables -A INPUT -s "192.168.2.32" -j ACCEPT
// iptables -A INPUT -s "192.168.2.32" -j REJECT
// iptables -D INPUT ...
// iptables -F INPUT
type IPTablesAction struct {
BaseAction
@@ -71,10 +75,16 @@ func (this *IPTablesAction) runAction(action string, listType IPListType, item *
}
func (this *IPTablesAction) runActionSingleIP(action string, listType IPListType, item *pb.IPItem) error {
// 暂时不支持ipv6
// TODO 将来支持ipv6
if utils.IsIPv6(item.IpFrom) {
return nil
}
if item.Type == "all" {
return nil
}
path := this.config.Path
var path = this.config.Path
var err error
if len(path) == 0 {
path, err = exec.LookPath("iptables")
@@ -85,6 +95,7 @@ func (this *IPTablesAction) runActionSingleIP(action string, listType IPListType
this.iptablesNotFound = true
return err
}
this.config.Path = path
}
iptablesAction := ""
switch action {
@@ -110,16 +121,15 @@ func (this *IPTablesAction) runActionSingleIP(action string, listType IPListType
return nil
}
cmd := exec.Command(path, args...)
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
var cmd = executils.NewTimeoutCmd(30*time.Second, path, args...)
cmd.WithStderr()
err = cmd.Run()
if err != nil {
output := stderr.Bytes()
if bytes.Contains(output, []byte("No chain/target/match")) {
var output = cmd.Stderr()
if strings.Contains(output, "No chain/target/match") {
err = nil
} else {
return errors.New(err.Error() + ", output: " + string(output))
return errors.New(err.Error() + ", output: " + output)
}
}
return nil

View File

@@ -68,7 +68,7 @@ func (this *ActionManager) UpdateActions(actions []*firewallconfigs.FirewallActi
remotelogs.Error("IPLIBRARY/ACTION_MANAGER", "action "+strconv.FormatInt(newAction.Id, 10)+", type:"+newAction.Type+": "+err.Error())
continue
}
if bytes.Compare(newConfigJSON, oldConfigJSON) != 0 {
if !bytes.Equal(newConfigJSON, oldConfigJSON) {
_ = oldInstance.Close()
// 重新创建

View File

@@ -1,13 +1,13 @@
package iplibrary
import (
"bytes"
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"os/exec"
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
"path/filepath"
"time"
)
// ScriptAction 脚本命令动作
@@ -45,25 +45,24 @@ func (this *ScriptAction) DeleteItem(listType IPListType, item *pb.IPItem) error
func (this *ScriptAction) runAction(action string, listType IPListType, item *pb.IPItem) error {
// TODO 智能支持 .sh 脚本文件
cmd := exec.Command(this.config.Path)
cmd.Env = []string{
var cmd = executils.NewTimeoutCmd(30*time.Second, this.config.Path)
cmd.WithEnv([]string{
"ACTION=" + action,
"TYPE=" + item.Type,
"IP_FROM=" + item.IpFrom,
"IP_TO=" + item.IpTo,
"EXPIRED_AT=" + fmt.Sprintf("%d", item.ExpiredAt),
"LIST_TYPE=" + listType,
}
})
if len(this.config.Cwd) > 0 {
cmd.Dir = this.config.Cwd
cmd.WithDir(this.config.Cwd)
} else {
cmd.Dir = filepath.Dir(this.config.Path)
cmd.WithDir(filepath.Dir(this.config.Path))
}
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
cmd.WithStderr()
err := cmd.Run()
if err != nil {
return errors.New(err.Error() + ", output: " + string(stderr.Bytes()))
return errors.New(err.Error() + ", output: " + cmd.Stderr())
}
return nil
}

View File

@@ -1,130 +0,0 @@
// 源码改自https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
package iplibrary
import (
"errors"
"io/ioutil"
"strconv"
"strings"
)
const (
IndexBlockLength = 12
)
var err error
type IP2Region struct {
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
lastIndexPtr int64
totalBlocks int64
dbData []byte
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
}
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
func getIpInfo(cityId int64, line []byte) *IpInfo {
lineSlice := strings.Split(string(line), "|")
ipInfo := &IpInfo{}
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
ipInfo.Country = lineSlice[0]
ipInfo.Region = lineSlice[1]
ipInfo.Province = lineSlice[2]
ipInfo.City = lineSlice[3]
ipInfo.ISP = lineSlice[4]
return ipInfo
}
func NewIP2Region(path string) (*IP2Region, error) {
var region = &IP2Region{}
region.dbData, err = ioutil.ReadFile(path)
if err != nil {
return nil, err
}
region.firstIndexPtr = region.ipLongAtOffset(0)
region.lastIndexPtr = region.ipLongAtOffset(4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
return region, nil
}
func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
ip, err := ip2long(ipStr)
if err != nil {
return nil, err
}
h := this.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m*IndexBlockLength
sip := this.ipLongAtOffset(p)
if ip < sip {
h = m - 1
} else {
eip := this.ipLongAtOffset(p + 4)
if ip > eip {
l = m + 1
} else {
dataPtr = this.ipLongAtOffset(p + 8)
break
}
}
}
if dataPtr == 0 {
return nil, nil
}
dataLen := (dataPtr >> 24) & 0xFF
dataPtr = dataPtr & 0x00FFFFFF
return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
}
func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
return int64(this.dbData[offset]) |
int64(this.dbData[offset+1])<<8 |
int64(this.dbData[offset+2])<<16 |
int64(this.dbData[offset+3])<<24
}
func ip2long(IpStr string) (int64, error) {
bits := strings.Split(IpStr, ".")
if len(bits) != 4 {
return 0, errors.New("ip format error")
}
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24-8*i)
}
return sum, nil
}

View File

@@ -79,7 +79,7 @@ func TestIPItem_Memory(t *testing.T) {
for i := 0; i < 2_000_000; i ++ {
list.Add(&IPItem{
Type: "ip",
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long("192.168.1.1"),
IPTo: 0,
ExpiredAt: time.Now().Unix(),

View File

@@ -3,31 +3,33 @@
package iplibrary
import (
"database/sql"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/iwind/TeaGo/Tea"
_ "github.com/mattn/go-sqlite3"
"os"
"path/filepath"
"time"
)
type IPListDB struct {
db *sql.DB
db *dbs.DB
itemTableName string
deleteExpiredItemsStmt *sql.Stmt
deleteItemStmt *sql.Stmt
insertItemStmt *sql.Stmt
selectItemsStmt *sql.Stmt
selectMaxVersionStmt *sql.Stmt
itemTableName string
deleteExpiredItemsStmt *dbs.Stmt
deleteItemStmt *dbs.Stmt
insertItemStmt *dbs.Stmt
selectItemsStmt *dbs.Stmt
selectMaxVersionStmt *dbs.Stmt
cleanTicker *time.Ticker
dir string
isClosed bool
}
func NewIPListDB() (*IPListDB, error) {
@@ -51,13 +53,29 @@ func (this *IPListDB) init() error {
remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = this.dir + "/ip_list.db"
db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
if err != nil {
return err
}
db.SetMaxOpenConns(1)
//_, err = db.Exec("VACUUM")
//if err != nil {
// return err
//}
this.db = db
// 恢复数据库
var recoverEnv, _ = os.LookupEnv("EdgeRecover")
if len(recoverEnv) > 0 {
for _, indexName := range []string{"ip_list_itemId", "ip_list_expiredAt"} {
_, _ = db.Exec(`REINDEX "` + indexName + `"`)
}
}
// 初始化数据库
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
@@ -117,6 +135,7 @@ ON "` + this.itemTableName + `" (
goman.New(func() {
events.On(events.EventQuit, func() {
_ = this.Close()
this.cleanTicker.Stop()
})
@@ -133,20 +152,38 @@ ON "` + this.itemTableName + `" (
// DeleteExpiredItems 删除过期的条目
func (this *IPListDB) DeleteExpiredItems() error {
if this.isClosed {
return nil
}
_, err := this.deleteExpiredItemsStmt.Exec(time.Now().Unix() - 7*86400)
return err
}
func (this *IPListDB) AddItem(item *pb.IPItem) error {
if this.isClosed {
return nil
}
_, err := this.deleteItemStmt.Exec(item.Id)
if err != nil {
return err
}
// 如果是删除,则不再创建新记录
if item.IsDeleted {
return nil
}
_, err = this.insertItemStmt.Exec(item.ListId, item.ListType, item.IsGlobal, item.Type, item.Id, item.IpFrom, item.IpTo, item.ExpiredAt, item.EventLevel, item.IsDeleted, item.Version, item.NodeId, item.ServerId)
return err
}
func (this *IPListDB) ReadItems(offset int64, size int64) (items []*pb.IPItem, err error) {
if this.isClosed {
return
}
rows, err := this.selectItemsStmt.Query(offset, size)
if err != nil {
return nil, err
@@ -169,12 +206,16 @@ func (this *IPListDB) ReadItems(offset int64, size int64) (items []*pb.IPItem, e
// ReadMaxVersion 读取当前最大版本号
func (this *IPListDB) ReadMaxVersion() int64 {
row := this.selectMaxVersionStmt.QueryRow()
if this.isClosed {
return 0
}
var row = this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err = row.Scan(&version)
err := row.Scan(&version)
if err != nil {
return 0
}
@@ -182,6 +223,8 @@ func (this *IPListDB) ReadMaxVersion() int64 {
}
func (this *IPListDB) Close() error {
this.isClosed = true
if this.db != nil {
_ = this.deleteExpiredItemsStmt.Close()
_ = this.deleteItemStmt.Close()

View File

@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.AddItem(&pb.IPItem{
Id: 1,
IpFrom: "192.168.1.101",
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.Close()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
@@ -53,6 +60,11 @@ func TestIPListDB_ReadItems(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer func() {
_ = db.Close()
}()
items, err := db.ReadItems(0, 2)
if err != nil {
t.Fatal(err)

View File

@@ -144,7 +144,7 @@ func TestIPList_Contains(t *testing.T) {
list := NewIPList()
for i := 0; i < 255; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0,
@@ -152,7 +152,7 @@ func TestIPList_Contains(t *testing.T) {
}
for i := 0; i < 255; i++ {
list.AddDelay(&IPItem{
Id: int64(1000 + i),
Id: uint64(1000 + i),
IPFrom: utils.IP2Long("192.167.2." + strconv.Itoa(i)),
})
}
@@ -172,7 +172,7 @@ func TestIPList_Contains_Many(t *testing.T) {
list := NewIPList()
for i := 0; i < 1_000_000; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255))),
ExpiredAt: 0,
@@ -217,7 +217,7 @@ func TestIPList_ContainsIPStrings(t *testing.T) {
list := NewIPList()
for i := 0; i < 255; i++ {
list.Add(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i) + ".168.0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i) + ".168.255.1"),
ExpiredAt: 0,
@@ -305,7 +305,7 @@ func BenchmarkIPList_Contains(b *testing.B) {
var list = NewIPList()
for i := 1; i < 200_000; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
Id: uint64(i),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60,

View File

@@ -1,13 +0,0 @@
package iplibrary
type LibraryInterface interface {
// Load 加载数据库文件
Load(dbPath string) error
// Lookup 查询IP
// 返回结果有可能为空
Lookup(ip string) (*Result, error)
// Close 关闭数据库文件
Close()
}

View File

@@ -1,83 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"net"
"strings"
)
type IP2RegionLibrary struct {
db *IP2Region
}
func (this *IP2RegionLibrary) Load(dbPath string) error {
db, err := NewIP2Region(dbPath)
if err != nil {
return err
}
this.db = db
return nil
}
func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
// 暂不支持IPv6
if strings.Contains(ip, ":") {
return nil, nil
}
if net.ParseIP(ip) == nil {
return nil, nil
}
if this.db == nil {
return nil, errors.New("library has not been loaded")
}
defer func() {
// 防止panic发生
err := recover()
if err != nil {
remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
}
}()
info, err := this.db.MemorySearch(ip)
if err != nil {
return nil, err
}
if info == nil {
return nil, nil
}
if info.Country == "0" {
info.Country = ""
}
if info.Region == "0" {
info.Region = ""
}
if info.Province == "0" {
info.Province = ""
}
if info.City == "0" {
info.City = ""
}
if info.ISP == "0" {
info.ISP = ""
}
return &Result{
CityId: info.CityId,
Country: info.Country,
Region: info.Region,
Province: info.Province,
City: info.City,
ISP: info.ISP,
}, nil
}
func (this *IP2RegionLibrary) Close() {
}

View File

@@ -1,113 +0,0 @@
package iplibrary
import (
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
"sync"
"testing"
"time"
)
func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
var mem = &runtime.MemStats{}
runtime.ReadMemStats(mem)
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var mem2 = &runtime.MemStats{}
runtime.ReadMemStats(mem2)
t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
}
func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"8.8.9.9"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
result, err := library.Lookup(ip)
if err != nil {
t.Fatal(err)
}
t.Log("IP:", ip, "result:", result)
}
}
func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
var count = 4000
var wg = sync.WaitGroup{}
wg.Add(count)
for i := 0; i < count; i++ {
go func() {
defer wg.Done()
for i := 0; i < 100; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
}()
}
wg.Done()
t.Log("ok")
}
func TestIP2RegionLibrary_Memory(t *testing.T) {
library := &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
t.Fatal(err)
}
before := time.Now()
for i := 0; i < 1_000_000; i++ {
_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
}
t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
}
func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
var library = &IP2RegionLibrary{}
err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_, _ = library.Lookup("8.8.8.8")
}
}

View File

@@ -1,95 +0,0 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/files"
"github.com/iwind/TeaGo/types"
"regexp"
"strings"
)
var SharedManager = NewManager()
var SharedLibrary LibraryInterface
func init() {
events.On(events.EventLoaded, func() {
// 初始化
library, err := SharedManager.Load()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
return
}
SharedLibrary = library
})
}
type Manager struct {
code string
}
func NewManager() *Manager {
return &Manager{}
}
func (this *Manager) Load() (LibraryInterface, error) {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return nil, err
}
config := nodeConfig.GlobalConfig
if config == nil {
config = &serverconfigs.GlobalConfig{}
}
// 当前正在使用的IP库代号
code := config.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
dir := Tea.Root + "/resources/ipdata/" + code
var lastVersion int64 = -1
lastFilename := ""
for _, file := range files.NewFile(dir).List() {
filename := file.Name()
reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
if reg.MatchString(filename) { // 先查找有版本号的
result := reg.FindStringSubmatch(filename)
version := types.Int64(result[1])
if version > lastVersion {
lastVersion = version
lastFilename = filename
}
} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
if lastVersion == -1 {
lastFilename = filename
lastVersion = 0
}
}
}
if len(lastFilename) == 0 {
return nil, errors.New("ip library file not found")
}
var libraryPtr LibraryInterface
switch code {
case serverconfigs.IPLibraryTypeIP2Region:
libraryPtr = &IP2RegionLibrary{}
default:
return nil, errors.New("invalid ip library code '" + code + "'")
}
err = libraryPtr.Load(dir + "/" + lastFilename)
if err != nil {
return nil, err
}
return libraryPtr, nil
}

View File

@@ -1,156 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/types"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCityManager = NewCityManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCityManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCityManager.Stop()
})
}
// CityManager 中国省份信息管理
type CityManager struct {
ticker *time.Ticker
cacheFile string
cityMap map[string]int64 // provinceName_cityName => cityName
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCityManager() *CityManager {
return &CityManager{
cacheFile: Tea.Root + "/configs/region_city.json.cache",
cityMap: map[string]int64{},
}
}
func (this *CityManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("City_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("CITY_MANAGER", err)
}
}
}
func (this *CityManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
this.locker.RLock()
cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *CityManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.cityMap = m
}
return nil
}
// 更新城市信息
func (this *CityManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCityRPC().FindAllEnabledRegionCities(rpcClient.Context(), &pb.FindAllEnabledRegionCitiesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, city := range resp.RegionCities {
for _, code := range city.Codes {
m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.cityMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,14 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewCityManager(t *testing.T) {
var manager = NewCityManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup(16, "许昌市"))
}

View File

@@ -1,154 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedCountryManager = NewCountryManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedCountryManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedCountryManager.Stop()
})
}
// CountryManager 国家/地区信息管理
type CountryManager struct {
ticker *time.Ticker
cacheFile string
countryMap map[string]int64 // countryName => countryId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewCountryManager() *CountryManager {
return &CountryManager{
cacheFile: Tea.Root + "/configs/region_country.json.cache",
countryMap: map[string]int64{},
}
}
func (this *CountryManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("COUNTRY_MANAGER", err)
}
}
}
func (this *CountryManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *CountryManager) Lookup(countryName string) (countryId int64) {
this.locker.RLock()
countryId, _ = this.countryMap[countryName]
this.locker.RUnlock()
return countryId
}
// 从缓存中读取
func (this *CountryManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.countryMap = m
}
return nil
}
// 更新国家信息
func (this *CountryManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionCountryRPC().FindAllEnabledRegionCountries(rpcClient.Context(), &pb.FindAllEnabledRegionCountriesRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, country := range resp.RegionCountries {
for _, code := range country.Codes {
m[code] = country.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.countryMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestCountryManager_load(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop(t *testing.T) {
manager := NewCountryManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.countryMap)
}
func TestCountryManager_loop_skip(t *testing.T) {
manager := NewCountryManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestCountryManager_Lookup(t *testing.T) {
manager := NewCountryManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
}
func BenchmarkCountryManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewCountryManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("中国")
}
}

View File

@@ -1,7 +1,9 @@
package iplibrary
import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -18,6 +20,10 @@ var SharedIPListManager = NewIPListManager()
var IPListUpdateNotify = make(chan bool, 1)
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedIPListManager.Start()
@@ -26,6 +32,13 @@ func init() {
events.On(events.EventQuit, func() {
SharedIPListManager.Stop()
})
var ticker = time.NewTicker(24 * time.Hour)
goman.New(func() {
for range ticker.C {
SharedIPListManager.DeleteExpiredItems()
}
})
}
// IPListManager IP名单管理
@@ -43,7 +56,7 @@ type IPListManager struct {
func NewIPListManager() *IPListManager {
return &IPListManager{
pageSize: 500,
pageSize: 1000,
listMap: map[int64]*IPList{},
}
}
@@ -111,20 +124,30 @@ func (this *IPListManager) init() {
var size int64 = 1000
for {
items, err := db.ReadItems(offset, size)
var l = len(items)
if err != nil {
remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
} else {
if len(items) == 0 {
if l == 0 {
break
}
this.processItems(items, false)
if int64(l) < size {
break
}
}
offset += int64(len(items))
offset += int64(l)
}
}
}
func (this *IPListManager) loop() error {
// 是否同步IP名单
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
if nodeConfig != nil && !nodeConfig.EnableIPLists {
return nil
}
for {
hasNext, err := this.fetch()
if err != nil {
@@ -144,7 +167,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
if err != nil {
return false, err
}
itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
Version: this.version,
Size: this.pageSize,
})
@@ -155,7 +178,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
}
return false, err
}
items := itemsResp.IpItems
var items = itemsResp.IpItems
if len(items) == 0 {
return false, nil
}
@@ -182,8 +205,13 @@ func (this *IPListManager) FindList(listId int64) *IPList {
return list
}
func (this *IPListManager) DeleteExpiredItems() {
if this.db != nil {
_ = this.db.DeleteExpiredItems()
}
}
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
this.locker.Lock()
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
var list *IPList
@@ -203,11 +231,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
list = GlobalWhiteIPList
}
} else { // 其他List
this.locker.Lock()
list = this.listMap[item.ListId]
this.locker.Unlock()
}
if list == nil {
list = NewIPList()
this.locker.Lock()
this.listMap[item.ListId] = list
this.locker.Unlock()
}
changedLists[list] = zero.New()
@@ -246,8 +278,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
changedList.Sort()
}
this.locker.Unlock()
if fromRemote {
var latestVersion = items[len(items)-1].Version
if latestVersion > this.version {

View File

@@ -1,155 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
var SharedProviderManager = NewProviderManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProviderManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProviderManager.Stop()
})
}
// ProviderManager 中国省份信息管理
type ProviderManager struct {
ticker *time.Ticker
cacheFile string
providerMap map[string]int64 // name => id
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProviderManager() *ProviderManager {
return &ProviderManager{
cacheFile: Tea.Root + "/configs/region_provider.json.cache",
providerMap: map[string]int64{},
}
}
func (this *ProviderManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVIDER_MANAGER", err)
}
}
}
func (this *ProviderManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
this.locker.RLock()
providerId, _ = this.providerMap[providerName]
this.locker.RUnlock()
return
}
// 从缓存中读取
func (this *ProviderManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.providerMap = m
}
return nil
}
// 更新服务商信息
func (this *ProviderManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProviderRPC().FindAllEnabledRegionProviders(rpcClient.Context(), &pb.FindAllEnabledRegionProvidersRequest{})
if err != nil {
return err
}
m := map[string]int64{}
for _, provider := range resp.RegionProviders {
for _, code := range provider.Codes {
m[code] = provider.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.providerMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,15 +0,0 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
import "testing"
func TestNewProviderManager(t *testing.T) {
var manager = NewProviderManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("阿里云"))
t.Log(manager.Lookup("阿里云2"))
}

View File

@@ -1,161 +0,0 @@
package iplibrary
import (
"crypto/md5"
"encoding/json"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
_ "github.com/iwind/TeaGo/bootstrap"
"io/ioutil"
"os"
"sync"
"time"
)
const (
ChinaCountryId int64 = 1
)
var SharedProvinceManager = NewProvinceManager()
func init() {
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedProvinceManager.Start()
})
})
events.On(events.EventQuit, func() {
SharedProvinceManager.Stop()
})
}
// ProvinceManager 中国省份信息管理
type ProvinceManager struct {
ticker *time.Ticker
cacheFile string
provinceMap map[string]int64 // provinceName => provinceId
dataHash string // 国家JSON的md5
locker sync.RWMutex
isUpdated bool
}
func NewProvinceManager() *ProvinceManager {
return &ProvinceManager{
cacheFile: Tea.Root + "/configs/region_province.json.cache",
provinceMap: map[string]int64{},
}
}
func (this *ProvinceManager) Start() {
// 从缓存中读取
err := this.load()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 第一次更新
err = this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
// 定时更新
this.ticker = time.NewTicker(4 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("PROVINCE_MANAGER", err)
}
}
}
func (this *ProvinceManager) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
this.locker.RLock()
provinceId, _ = this.provinceMap[provinceName]
this.locker.RUnlock()
return provinceId
}
// 从缓存中读取
func (this *ProvinceManager) load() error {
data, err := ioutil.ReadFile(this.cacheFile)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
m := map[string]int64{}
err = json.Unmarshal(data, &m)
if err != nil {
return err
}
if m != nil && len(m) > 0 {
this.provinceMap = m
}
return nil
}
// 更新省份信息
func (this *ProvinceManager) loop() error {
if this.isUpdated {
return nil
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.RegionProvinceRPC().FindAllEnabledRegionProvincesWithCountryId(rpcClient.Context(), &pb.FindAllEnabledRegionProvincesWithCountryIdRequest{
RegionCountryId: ChinaCountryId,
})
if err != nil {
return err
}
m := map[string]int64{}
for _, province := range resp.RegionProvinces {
for _, code := range province.Codes {
m[code] = province.Id
}
}
// 检查是否有更新
data, err := json.Marshal(m)
if err != nil {
return err
}
hash := md5.New()
hash.Write(data)
dataHash := fmt.Sprintf("%x", hash.Sum(nil))
if this.dataHash == dataHash {
return nil
}
this.dataHash = dataHash
this.locker.Lock()
this.provinceMap = m
this.isUpdated = true
this.locker.Unlock()
// 保存到本地缓存
err = ioutil.WriteFile(this.cacheFile, data, 0666)
return err
}

View File

@@ -1,57 +0,0 @@
package iplibrary
import (
"runtime"
"testing"
)
func TestProvinceManager_load(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop(t *testing.T) {
manager := NewProvinceManager()
err := manager.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok", manager.provinceMap)
}
func TestProvinceManager_loop_skip(t *testing.T) {
manager := NewProvinceManager()
for i := 0; i < 10; i++ {
err := manager.loop()
if err != nil {
t.Fatal(err)
}
}
}
func TestProvinceManager_Lookup(t *testing.T) {
manager := NewProvinceManager()
err := manager.load()
if err != nil {
t.Fatal(err)
}
t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
}
func BenchmarkProvinceManager_Lookup(b *testing.B) {
runtime.GOMAXPROCS(1)
manager := NewProvinceManager()
err := manager.load()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_ = manager.Lookup("安徽省")
}
}

View File

@@ -1,26 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestManager_Load(t *testing.T) {
dbs.NotifyReady()
manager := NewManager()
lib, err := manager.Load()
if err != nil {
t.Fatal(err)
}
t.Log(lib.Lookup("1.2.3.4"))
t.Log(lib.Lookup("2.3.4.5"))
t.Log(lib.Lookup("200.200.200.200"))
t.Log(lib.Lookup("202.106.0.20"))
}
func TestNewManager(t *testing.T) {
dbs.NotifyReady()
t.Log(SharedLibrary)
}

View File

@@ -1,153 +0,0 @@
package iplibrary
import (
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/errors"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"os"
"time"
)
var SharedUpdater = NewUpdater()
func init() {
events.On(events.EventStart, func() {
goman.New(func() {
SharedUpdater.Start()
})
})
events.On(events.EventQuit, func() {
SharedUpdater.Stop()
})
}
// Updater IP库更新程序
type Updater struct {
ticker *time.Ticker
}
// NewUpdater 获取新对象
func NewUpdater() *Updater {
return &Updater{}
}
// Start 开始更新
func (this *Updater) Start() {
// 这里不需要太频繁检查更新因为通常不需要更新IP库
this.ticker = time.NewTicker(1 * time.Hour)
for range this.ticker.C {
err := this.loop()
if err != nil {
remotelogs.ErrorObject("IP_LIBRARY", err)
}
}
}
func (this *Updater) Stop() {
if this.ticker != nil {
this.ticker.Stop()
}
}
// 单次任务
func (this *Updater) loop() error {
nodeConfig, err := nodeconfigs.SharedNodeConfig()
if err != nil {
return err
}
if nodeConfig.GlobalConfig == nil {
return nil
}
code := nodeConfig.GlobalConfig.IPLibrary.Code
if len(code) == 0 {
code = serverconfigs.DefaultIPLibraryType
}
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
if err != nil {
return err
}
lib := libraryResp.IpLibrary
if lib == nil || lib.File == nil {
return nil
}
typeInfo := serverconfigs.FindIPLibraryWithType(code)
if typeInfo == nil {
return errors.New("invalid ip library code '" + code + "'")
}
path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
// 是否已经存在
_, err = os.Stat(path)
if err == nil {
return nil
}
// 开始下载
fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
if err != nil {
return err
}
chunkIds := fileChunkIdsResp.FileChunkIds
if len(chunkIds) == 0 {
return nil
}
isOk := false
fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer func() {
// 如果保存不成功就直接删除
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
}
}()
for _, chunkId := range chunkIds {
chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
if err != nil {
return err
}
chunk := chunkResp.FileChunk
if chunk == nil {
continue
}
_, err = fp.Write(chunk.Data)
if err != nil {
return err
}
}
err = fp.Close()
if err != nil {
return err
}
// 重新加载
library, err := SharedManager.Load()
if err != nil {
return err
}
SharedLibrary = library
isOk = true
return nil
}

View File

@@ -1,18 +0,0 @@
package iplibrary
import (
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/dbs"
"testing"
)
func TestUpdater_loop(t *testing.T) {
dbs.NotifyReady()
updater := NewUpdater()
err := updater.loop()
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}

View File

@@ -4,6 +4,8 @@ package metrics
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"strconv"
"sync"
@@ -11,7 +13,19 @@ import (
var SharedManager = NewManager()
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventQuit, func() {
SharedManager.Quit()
})
}
type Manager struct {
isQuiting bool
tasks map[int64]*Task // itemId => *Task
categoryTasks map[string][]*Task // category => []*Task
locker sync.RWMutex
@@ -29,6 +43,10 @@ func NewManager() *Manager {
}
func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
if this.isQuiting {
return
}
this.locker.Lock()
defer this.locker.Unlock()
@@ -101,6 +119,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
// Add 添加数据
func (this *Manager) Add(obj MetricInterface) {
if this.isQuiting {
return
}
this.locker.RLock()
for _, task := range this.categoryTasks[obj.MetricCategory()] {
task.Add(obj)
@@ -119,3 +141,17 @@ func (this *Manager) HasTCPMetrics() bool {
func (this *Manager) HasUDPMetrics() bool {
return this.hasUDPMetrics
}
// Quit 退出管理器
func (this *Manager) Quit() {
this.isQuiting = true
remotelogs.Println("METRIC_MANAGER", "quit")
this.locker.Lock()
for _, task := range this.tasks {
_ = task.Stop()
}
this.tasks = map[int64]*Task{}
this.locker.Unlock()
}

View File

@@ -3,7 +3,6 @@
package metrics
import (
"database/sql"
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
@@ -16,25 +15,26 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
_ "github.com/mattn/go-sqlite3"
"github.com/iwind/TeaGo/types"
"os"
"strconv"
"sync"
"time"
)
const MaxQueueSize = 2048
const MaxQueueSize = 256 // TODO 可以配置,可以在单个任务里配置
// Task 单个指标任务
// 数据库存储:
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 上传每个服务中排序最前面的 N 个数据
//
// data/
// metric.$ID.db
// stats
// id, keys, value, time, serverId, hash
// 原理:
// 添加或者有变更时 isUploaded = false
// 上传时检查 isUploaded 状态
// 只上传每个服务中排序最前面的 N 个数据
type Task struct {
item *serverconfigs.MetricItemConfig
isLoaded bool
@@ -48,11 +48,11 @@ type Task struct {
cleanVersion int32
insertStatStmt *sql.Stmt
deleteByVersionStmt *sql.Stmt
deleteByExpiresTimeStmt *sql.Stmt
selectTopStmt *sql.Stmt
sumStmt *sql.Stmt
insertStatStmt *dbs.Stmt
deleteByVersionStmt *dbs.Stmt
deleteByExpiresTimeStmt *dbs.Stmt
selectTopStmt *dbs.Stmt
sumStmt *dbs.Stmt
serverIdMap map[int64]zero.Zero // 所有的服务Ids
timeMap map[string]zero.Zero // time => bool
@@ -88,12 +88,22 @@ func (this *Task) Init() error {
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
var path = dir + "/metric." + types.String(this.item.Id) + ".db"
db, err := dbs.OpenWriter("file:" + path + "?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF&_locking_mode=EXCLUSIVE")
if err != nil {
return err
}
db.SetMaxOpenConns(1)
this.db = dbs.NewDB(db)
this.db = db
// 恢复数据库
var recoverEnv, _ = os.LookupEnv("EdgeRecover")
if len(recoverEnv) > 0 {
for _, indexName := range []string{"serverId", "hash"} {
_, _ = db.Exec(`REINDEX "` + indexName + `"`)
}
}
if teaconst.EnableDBStat {
this.db.EnableStat(true)
@@ -372,7 +382,9 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
for _, serverId := range serverIds {
for _, currentTime := range times {
idStrings, err := func(serverId int64, currentTime string) (ids []string, err error) {
var t = trackers.Begin("[METRIC]SELECT_TOP_STMT")
rows, err := this.selectTopStmt.Query(serverId, this.item.Version, currentTime)
t.End()
if err != nil {
return nil, err
}
@@ -421,7 +433,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
return nil, err
}
_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
MetricStats: pbStats,
Time: currentTime,
ServerId: serverId,

View File

@@ -5,6 +5,7 @@ package monitor
import (
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -16,6 +17,10 @@ import (
var SharedValueQueue = NewValueQueue()
func init() {
if !teaconst.IsMain {
return
}
events.On(events.EventLoaded, func() {
goman.New(func() {
SharedValueQueue.Start()
@@ -69,7 +74,7 @@ func (this *ValueQueue) Loop() error {
}
for value := range this.valuesChan {
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
Item: value.Item,
ValueJSON: value.ValueJSON,
CreatedAt: value.CreatedAt,

View File

@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
if err != nil {
statusErr, ok:= status.FromError(err)
statusErr, ok := status.FromError(err)
if ok {
logs.Println(statusErr.Code())
}

Some files were not shown because too many files have changed in this diff Show More