Compare commits

..

60 Commits

Author SHA1 Message Date
刘祥超
130519be71 增加测试用例 2022-03-20 18:33:51 +08:00
刘祥超
da0fdf6485 优化代码 2022-03-20 16:20:40 +08:00
刘祥超
37f3e6fa2d 取消查询缓存超时的设置 2022-03-20 16:12:49 +08:00
刘祥超
c2151e895d GRPC通讯支持gzip压缩 2022-03-20 11:28:34 +08:00
刘祥超
e15afc0e0c 修复一个测试用例 2022-03-20 10:53:35 +08:00
刘祥超
c86f2d5a44 优化代码 2022-03-20 10:48:11 +08:00
刘祥超
d45254b46d 更新相关库 2022-03-20 10:48:06 +08:00
刘祥超
83df9d0c29 内容压缩算法使用Pool管理 2022-03-20 00:05:47 +08:00
刘祥超
856f9d4a3b 调整Brotli压缩参数 2022-03-19 19:28:20 +08:00
刘祥超
14eda8c276 优化代码 2022-03-19 12:17:28 +08:00
刘祥超
0e732e4821 OCSP支持过期时间 2022-03-18 20:20:47 +08:00
刘祥超
b5b7ab99d3 动态更新OCSP,防止过期 2022-03-18 17:09:15 +08:00
刘祥超
134b56c87a 优化代码 2022-03-17 19:48:04 +08:00
刘祥超
a381850dbc 源站可以自定义回源主机名 2022-03-17 17:03:52 +08:00
刘祥超
14abce08c3 IPSet支持IPv6 2022-03-16 20:47:55 +08:00
刘祥超
469a3ea979 优化WAF性能 2022-03-16 17:06:26 +08:00
刘祥超
9b85487a70 提升缓存效率 2022-03-16 16:20:53 +08:00
刘祥超
06ef206c9f 节点可以单独设置缓存目录 2022-03-16 15:24:35 +08:00
刘祥超
1c2ca73208 缓存策略修改时尽可能不重新加载 2022-03-15 21:33:44 +08:00
刘祥超
b9abc55728 本地数据库升级之后增加日志 2022-03-15 18:58:56 +08:00
刘祥超
21e206061d 优化本地数据库性能 2022-03-15 18:32:39 +08:00
刘祥超
ddebc0e4a8 增加回源跟随功能 2022-03-14 15:07:18 +08:00
刘祥超
31b16ad67a 正在退出时不上报错误 2022-03-14 12:23:28 +08:00
刘祥超
d0b86af4ef 程序意外退出时关闭sqlite指针 2022-03-14 11:47:34 +08:00
刘祥超
ce87fa25e7 sqlite添加参数_sync=OFF 2022-03-14 11:08:02 +08:00
刘祥超
ecbb11eee2 增加对数据库操作的统计命令:edge-node dbstat/减少几个不必要的查询操作 2022-03-13 19:27:38 +08:00
刘祥超
9849f14044 自动为热点数据设置合适的过期时间 2022-03-12 20:50:05 +08:00
刘祥超
34fae1c2a3 内存缓存刷到磁盘后自动从内存中删除 2022-03-12 20:01:28 +08:00
刘祥超
8b22d00ce4 优化代码 2022-03-12 19:45:22 +08:00
刘祥超
1e64386744 更新相关库 2022-03-12 19:11:29 +08:00
刘祥超
8dfeda0d80 优化代码 2022-03-12 18:00:22 +08:00
刘祥超
0ece9e0897 优化代码 2022-03-11 17:21:23 +08:00
刘祥超
79ffda4706 使用异步IO写入缓存文件 2022-03-11 15:29:18 +08:00
刘祥超
0d2d7591e5 自动清理本地IP名单过期条目/修复白名单可能不起作用的Bug 2022-03-06 19:40:26 +08:00
刘祥超
577a5618a1 修复纯内存缓存可能启动多个flush内容的goroutine的Bug 2022-03-06 18:09:33 +08:00
刘祥超
49822ab7e9 分块传输内容可以写入到内存中/分块传输内容可以判断最大尺寸 2022-03-06 17:18:06 +08:00
刘祥超
421cb82505 增加Partial Content大尺寸测试用例 2022-03-06 14:40:31 +08:00
刘祥超
1c0192d773 优化Partial Content缓存 2022-03-05 19:31:50 +08:00
刘祥超
10443dfb15 优化Partial Content缓存 2022-03-05 16:47:17 +08:00
刘祥超
f34ad57e12 优化Partial Content缓存 2022-03-04 22:42:03 +08:00
刘祥超
21c80aea78 不支持对GET以外的方法返回的Partial内容的缓存 2022-03-04 17:09:12 +08:00
刘祥超
16c123c400 当缓存条件状态码为206时,自动支持区间缓存 2022-03-04 17:00:48 +08:00
刘祥超
18d301e17f 更新TeaGo 2022-03-04 15:11:18 +08:00
刘祥超
dc9a399ff8 升级相关依赖库 2022-03-04 12:30:06 +08:00
刘祥超
269e33b9a0 删除Partial缓存时,同时删除区间范围相关文件 2022-03-04 11:51:59 +08:00
刘祥超
581a3d49fc 实现基础的206 partial content缓存 2022-03-03 19:36:28 +08:00
刘祥超
8aeb5eb1b6 WAF有允许(allow)动作出现的时候不再继续往下执行 2022-02-26 17:13:27 +08:00
刘祥超
4d79ce7659 修改Partial Content的Bounary值长度(从60字节修改为16字节) 2022-02-25 19:09:09 +08:00
刘祥超
9fda8c425a 增加 edge-node reload 命令/优化命令行帮助 2022-02-25 11:23:32 +08:00
刘祥超
14259b2ef5 Update http_request_cache.go 2022-02-24 20:39:43 +08:00
刘祥超
211750eb8e 默认不同步写入压缩缓存/增加是否同步写入压缩缓存设置/去除默认content-type类型设置 2022-02-24 20:13:05 +08:00
刘祥超
1050231086 修改版本为0.4.5 2022-02-24 19:26:57 +08:00
刘祥超
034965a697 优化代码 2022-02-24 16:44:28 +08:00
刘祥超
239acf2d17 修复WAF提取正则表达式关键词时可能会使程序崩溃的Bug 2022-02-23 17:35:35 +08:00
刘祥超
6260fd1009 sqlite缓存表增加tag字段 2022-02-23 16:06:18 +08:00
刘祥超
f10ffbaebd 修改版本为0.4.4 2022-02-23 14:49:42 +08:00
刘祥超
83f581f7f0 清理缓存时也清理HEAD缓存 2022-02-22 21:52:28 +08:00
刘祥超
03ce082926 支持对GET/POST之外的请求方法独立缓存 2022-02-22 21:43:47 +08:00
刘祥超
ebafe458ff 支持缓存压缩后的内容 2022-02-22 19:29:27 +08:00
刘祥超
ff30b8d15c 修改版本为v0.4.3 2022-02-21 18:31:17 +08:00
143 changed files with 5282 additions and 1534 deletions

View File

@@ -22,7 +22,9 @@ func main() {
app := apps.NewAppCmd().
Version(teaconst.Version).
Product(teaconst.ProductName).
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|service|daemon|pprof]")
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof]").
Usage(teaconst.ProcessName + " [trackers|goman|conns|gc]").
Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove] IP")
app.On("test", func() {
err := nodes.NewNode().Test()
@@ -30,6 +32,20 @@ func main() {
_, _ = os.Stderr.WriteString(err.Error())
}
})
app.On("reload", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{Code: "reload"})
if err != nil {
fmt.Println("[ERROR]" + err.Error())
} else {
var params = maps.NewMap(reply.Params)
if params.Has("error") {
fmt.Println("[ERROR]" + params.GetString("error"))
} else {
fmt.Println("ok")
}
}
})
app.On("daemon", func() {
nodes.NewNode().Daemon()
})
@@ -65,6 +81,12 @@ func main() {
node := nodes.NewNode()
node.Start()
})
app.On("dbstat", func() {
teaconst.EnableDBStat = true
node := nodes.NewNode()
node.Start()
})
app.On("trackers", func() {
var sock = gosock.NewTmpSock(teaconst.ProcessName)
reply, err := sock.Send(&gosock.Command{Code: "trackers"})

20
go.mod
View File

@@ -8,15 +8,14 @@ replace (
require (
github.com/TeaOSLab/EdgeCommon v0.0.0-00010101000000-000000000000
github.com/andybalholm/brotli v1.0.3
github.com/andybalholm/brotli v1.0.4
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670
github.com/cespare/xxhash v1.1.0
github.com/chai2010/webp v1.1.0 // indirect
github.com/dchest/captcha v0.0.0-20200903113550-03f5f0333e1f
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/fsnotify/fsnotify v1.5.1
github.com/golang/protobuf v1.5.2
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
github.com/iwind/gowebp v0.0.0-20211029040624-7331ecc78ed8
@@ -26,15 +25,12 @@ require (
github.com/miekg/dns v1.1.43
github.com/mssola/user_agent v0.5.3
github.com/pires/go-proxyproto v0.6.1
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/tklauser/go-sysconf v0.3.6 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/shirou/gopsutil/v3 v3.22.2
golang.org/x/image v0.0.0-20211028202545-6944b10bf410
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
golang.org/x/text v0.3.7
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/grpc v1.43.0
google.golang.org/grpc v1.45.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

71
go.sum
View File

@@ -6,8 +6,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/biessek/golang-ico v0.0.0-20180326222316-d348d9ea4670 h1:FQPKKjDhzG0T4ew6dm6MGrXb4PRAi8ZmTuYuxcF62BM=
@@ -23,7 +23,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -38,7 +37,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -52,8 +50,6 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
github.com/go-redis/redis/v8 v8.0.0-beta.7/go.mod h1:FGJAWDWFht1sQ4qxyJHZZbVyvnVcKQN0E3u5/5lRz+g=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -75,15 +71,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iwind/TeaGo v0.0.0-20210628135026-38575a4ab060/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24 h1:1cGulkD2SNJJRok5OKwyhP/Ddm+PgSWKOupn0cR36/A=
github.com/iwind/TeaGo v0.0.0-20211026123858-7de7a21cad24/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475 h1:EseyfFaQOjWanGiby9KMw7PjDBMg/95tLDgIw/ns0Cw=
github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11 h1:DaQjoWZhLNxjhIXedVg4/vFEtHkZhK4IjIwsWdyzBLg=
github.com/iwind/gofcgi v0.0.0-20210528023741-a92711d45f11/go.mod h1:JtbX20untAjUVjZs1ZBtq80f5rJWvwtQNRL6EnuYRnY=
github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
@@ -100,6 +97,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
@@ -122,10 +121,12 @@ github.com/pires/go-proxyproto v0.6.1 h1:EBupykFmo22SDjv4fQVQd2J9NOoLPmyZA/15ldO
github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -134,10 +135,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/otel v0.7.0/go.mod h1:aZMyHG5TqDOXEgH2tyLiXSUKly1jT3yqE9PmrzIeCdo=
@@ -171,8 +172,8 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9 h1:kmreh1vGI63l2FxOAYS3Yv6ATsi7lSTuwNSVbGfJV9I=
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -195,17 +196,21 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d h1:1oIt9o40TWWI9FUaveVpUvBe13FNqBNVXy3ue2fcfkw=
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -230,19 +235,21 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0=
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220303160752-862486edd9cc h1:fb/ViRpv3ln/LvbqZtTpoOd1YQDNH12gaGZreoSFovE=
google.golang.org/genproto v0.0.0-20220303160752-862486edd9cc/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e h1:fNKDNuUyC4WH+inqDMpfXDdfvwfYILbsX+oskGZ8hxg=
google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -269,10 +276,8 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -20,7 +20,7 @@ import (
type AppCmd struct {
product string
version string
usage string
usages []string
options []*CommandHelpOption
appendStrings []string
@@ -54,7 +54,7 @@ func (this *AppCmd) Version(version string) *AppCmd {
// Usage 使用方法
func (this *AppCmd) Usage(usage string) *AppCmd {
this.usage = usage
this.usages = append(this.usages, usage)
return this
}
@@ -77,8 +77,10 @@ func (this *AppCmd) Append(appendString string) *AppCmd {
func (this *AppCmd) Print() {
fmt.Println(this.product + " v" + this.version)
usage := this.usage
fmt.Println("Usage:", "\n "+usage)
fmt.Println("Usage:")
for _, usage := range this.usages {
fmt.Println(" " + usage)
}
if len(this.options) > 0 {
fmt.Println("")

10
internal/caches/consts.go Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
const (
SuffixWebP = "@GOEDGE_WEBP" // WebP后缀
SuffixCompression = "@GOEDGE_" // 压缩后缀 SuffixCompression + Encoding
SuffixMethod = "@GOEDGE_" // 请求方法后缀 SuffixMethod + RequestMethod
SuffixPartial = "@GOEDGE_partial" // 分区缓存后缀
)

View File

@@ -6,9 +6,12 @@ import "errors"
// 常用的几个错误
var (
ErrNotFound = errors.New("cache not found")
ErrFileIsWriting = errors.New("the file is writing")
ErrInvalidRange = errors.New("invalid range")
ErrNotFound = errors.New("cache not found")
ErrFileIsWriting = errors.New("the file is writing")
ErrInvalidRange = errors.New("invalid range")
ErrEntityTooLarge = errors.New("entity too large")
ErrWritingUnavailable = errors.New("writing unavailable")
ErrWritingQueueFull = errors.New("writing queue full")
)
// CapacityError 容量错误
@@ -30,7 +33,10 @@ func CanIgnoreErr(err error) bool {
if err == nil {
return true
}
if err == ErrFileIsWriting {
if err == ErrFileIsWriting ||
err == ErrEntityTooLarge ||
err == ErrWritingUnavailable ||
err == ErrWritingQueueFull {
return true
}
_, ok := err.(*CapacityError)

View File

@@ -3,8 +3,6 @@
package caches
type HotItem struct {
Key string
ExpiresAt int64
Hits uint32
Status int
Key string
Hits uint32
}

View File

@@ -4,52 +4,32 @@ package caches
import (
"database/sql"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/lists"
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
_ "github.com/mattn/go-sqlite3"
"os"
"sync/atomic"
"time"
)
const CountFileDB = 20
// FileList 文件缓存列表管理
type FileList struct {
dir string
db *sql.DB
total int64
dir string
dbList [CountFileDB]*FileListDB
total int64
onAdd func(item *Item)
onRemove func(item *Item)
// cacheItems
existsByHashStmt *sql.Stmt // 根据hash检查是否存在
insertStmt *sql.Stmt // 写入数据
selectByHashStmt *sql.Stmt // 使用hash查询数据
deleteByHashStmt *sql.Stmt // 根据hash删除数据
statStmt *sql.Stmt // 统计
purgeStmt *sql.Stmt // 清理
deleteAllStmt *sql.Stmt // 删除所有数据
// hits
insertHitStmt *sql.Stmt // 写入数据
increaseHitStmt *sql.Stmt // 增加点击量
deleteHitByHashStmt *sql.Stmt // 根据hash删除数据
lfuHitsStmt *sql.Stmt // 读取老的数据
oldTables []string
itemsTableName string
hitsTableName string
isClosed bool
isReady bool
memoryCache *ttlcache.Cache
// 老数据库地址
oldDir string
}
func NewFileList(dir string) ListInterface {
@@ -59,6 +39,10 @@ func NewFileList(dir string) ListInterface {
}
}
func (this *FileList) SetOldDir(oldDir string) {
this.oldDir = oldDir
}
func (this *FileList) Init() error {
// 检查目录是否存在
_, err := os.Stat(this.dir)
@@ -70,114 +54,38 @@ func (this *FileList) Init() error {
remotelogs.Println("CACHE", "create cache dir '"+this.dir+"'")
}
this.itemsTableName = "cacheItems_v3"
this.hitsTableName = "hits"
var dir = this.dir
if dir == "/" {
// 防止sqlite提示authority错误
dir = ""
}
var dbPath = dir + "/index.db"
remotelogs.Println("CACHE", "loading database '"+dbPath+"'")
db, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=shared&mode=rwc&_journal_mode=WAL")
if err != nil {
return errors.New("open database failed: " + err.Error())
}
db.SetMaxOpenConns(1)
remotelogs.Println("CACHE", "loading database from '"+dir+"' ...")
for i := 0; i < CountFileDB; i++ {
var db = NewFileListDB()
err = db.Open(dir + "/db-" + types.String(i) + ".db")
if err != nil {
return err
}
this.db = db
err = db.Init()
if err != nil {
return err
}
// TODO 耗时过长,暂时不整理数据库
/**_, err = db.Exec("VACUUM")
if err != nil {
return err
}**/
// 创建
err = this.initTables(db, 1)
if err != nil {
return errors.New("init tables failed: " + err.Error())
}
// 清除旧表
// 这个一定要在initTables()之后,因为老的数据需要转移
this.oldTables = []string{
"cacheItems",
"cacheItems_v2",
}
err = this.removeOldTables()
if err != nil {
remotelogs.Warn("CACHE", "clean old tables failed: "+err.Error())
this.dbList[i] = db
}
// 读取总数量
row := this.db.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
if row.Err() != nil {
return row.Err()
}
var total int64
err = row.Scan(&total)
if err != nil {
return err
}
this.total = total
// 常用语句
this.existsByHashStmt, err = this.db.Prepare(`SELECT "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? AND expiredAt>? LIMIT 1`)
if err != nil {
return err
this.total = 0
for _, db := range this.dbList {
this.total += db.total
}
this.insertStmt, err = this.db.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
this.selectByHashStmt, err = this.db.Prepare(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? LIMIT 1`)
if err != nil {
return err
}
this.deleteByHashStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.statStmt, err = this.db.Prepare(`SELECT COUNT(*), IFNULL(SUM(headerSize+bodySize+metaSize), 0), IFNULL(SUM(headerSize+bodySize), 0) FROM "` + this.itemsTableName + `"`)
if err != nil {
return err
}
this.purgeStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" WHERE staleAt<=? LIMIT ?`)
if err != nil {
return err
}
this.deleteAllStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemsTableName + `"`)
if err != nil {
return err
}
this.insertHitStmt, err = this.db.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.increaseHitStmt, err = this.db.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
if err != nil {
return err
}
this.deleteHitByHashStmt, err = this.db.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.lfuHitsStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
return err
}
this.isReady = true
// 升级老版本数据库
goman.New(func() {
this.upgradeOldDB()
})
return nil
}
@@ -188,26 +96,22 @@ func (this *FileList) Reset() error {
}
func (this *FileList) Add(hash string, item *Item) error {
if !this.isReady {
var db = this.getDB(hash)
if !db.IsReady() {
return nil
}
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
err := db.Add(hash, item)
if err != nil {
return err
}
_, err = this.insertHitStmt.Exec(hash, timeutil.Format("YW"))
if err != nil {
return err
}
atomic.AddInt64(&this.total, 1)
// 这里不增加点击量,以减少对数据库的操作次数
this.memoryCache.Write(hash, 1, item.ExpiredAt)
atomic.AddInt64(&this.total, 1)
if this.onAdd != nil {
this.onAdd(item)
@@ -216,40 +120,36 @@ func (this *FileList) Add(hash string, item *Item) error {
}
func (this *FileList) Exist(hash string) (bool, error) {
if !this.isReady {
var db = this.getDB(hash)
if !db.IsReady() {
return false, nil
}
item := this.memoryCache.Read(hash)
var item = this.memoryCache.Read(hash)
if item != nil {
return true, nil
}
rows, err := this.existsByHashStmt.Query(hash, time.Now().Unix())
var row = db.existsByHashStmt.QueryRow(hash, time.Now().Unix())
if row.Err() != nil {
return false, nil
}
var expiredAt int64
err := row.Scan(&expiredAt)
if err != nil {
if err == sql.ErrNoRows {
err = nil
}
return false, err
}
defer func() {
_ = rows.Close()
}()
if rows.Next() {
var expiredAt int64
err = rows.Scan(&expiredAt)
if err != nil {
return false, nil
}
this.memoryCache.Write(hash, 1, expiredAt)
return true, nil
}
return false, nil
this.memoryCache.Write(hash, 1, expiredAt)
return true, nil
}
// CleanPrefix 清理某个前缀的缓存数据
func (this *FileList) CleanPrefix(prefix string) error {
if !this.isReady {
return nil
}
if len(prefix) == 0 {
return nil
}
@@ -258,106 +158,48 @@ func (this *FileList) CleanPrefix(prefix string) error {
this.memoryCache.Clean()
}()
var count = int64(10000)
var staleLife = 600 // TODO 需要可以设置
var unixTime = utils.UnixTime() // 只删除当前的,不删除新的
for {
result, err := this.db.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
for _, db := range this.dbList {
err := db.CleanPrefix(prefix)
if err != nil {
return err
}
affectedRows, err := result.RowsAffected()
if err != nil {
return err
}
if affectedRows < count {
return nil
}
}
return nil
}
func (this *FileList) Remove(hash string) error {
if !this.isReady {
return nil
}
// 从缓存中删除
this.memoryCache.Delete(hash)
row := this.selectByHashStmt.QueryRow(hash)
if row.Err() != nil {
return row.Err()
}
var item = &Item{Type: ItemTypeFile}
err := row.Scan(&item.Key, &item.HeaderSize, &item.BodySize, &item.MetaSize, &item.ExpiredAt)
if err != nil {
if err == sql.ErrNoRows {
return nil
}
return err
}
_, err = this.deleteByHashStmt.Exec(hash)
if err != nil {
return err
}
_, err = this.deleteHitByHashStmt.Exec(hash)
if err != nil {
return err
}
atomic.AddInt64(&this.total, -1)
if this.onRemove != nil {
this.onRemove(item)
}
return nil
_, err := this.remove(hash)
return err
}
// Purge 清理过期的缓存
// count 每次遍历的最大数量,控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用
func (this *FileList) Purge(count int, callback func(hash string) error) (int, error) {
if !this.isReady {
return 0, nil
}
count /= CountFileDB
if count <= 0 {
count = 1000
count = 100
}
rows, err := this.purgeStmt.Query(time.Now().Unix(), count)
if err != nil {
return 0, err
}
hashStrings := []string{}
var countFound = 0
for rows.Next() {
var hash string
err = rows.Scan(&hash)
for _, db := range this.dbList {
hashStrings, err := db.ListExpiredItems(count)
if err != nil {
_ = rows.Close()
return 0, err
return 0, nil
}
hashStrings = append(hashStrings, hash)
countFound++
}
_ = rows.Close() // 不能使用defer防止读写冲突
countFound += len(hashStrings)
// 不在 rows.Next() 循环中操作是为了避免死锁
for _, hash := range hashStrings {
err = this.Remove(hash)
if err != nil {
return 0, err
}
// 不在 rows.Next() 循环中操作是为了避免死锁
for _, hash := range hashStrings {
err = this.Remove(hash)
if err != nil {
return 0, err
}
err = callback(hash)
if err != nil {
return 0, err
err = callback(hash)
if err != nil {
return 0, err
}
}
}
@@ -365,80 +207,80 @@ func (this *FileList) Purge(count int, callback func(hash string) error) (int, e
}
func (this *FileList) PurgeLFU(count int, callback func(hash string) error) error {
if !this.isReady {
return nil
}
count /= CountFileDB
if count <= 0 {
return nil
count = 100
}
rows, err := this.lfuHitsStmt.Query(count)
if err != nil {
return err
}
hashStrings := []string{}
var countFound = 0
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
_ = rows.Close()
return err
}
hashStrings = append(hashStrings, hash)
countFound++
}
_ = rows.Close() // 不能使用defer防止读写冲突
// 不在 rows.Next() 循环中操作是为了避免死锁
for _, hash := range hashStrings {
err = this.Remove(hash)
for _, db := range this.dbList {
hashStrings, err := db.ListLFUItems(count)
if err != nil {
return err
}
err = callback(hash)
if err != nil {
return err
// 不在 rows.Next() 循环中操作是为了避免死锁
for _, hash := range hashStrings {
notFound, err := this.remove(hash)
if err != nil {
return err
}
if notFound {
_, err = db.deleteHitByHashStmt.Exec(hash)
if err != nil {
return err
}
}
err = callback(hash)
if err != nil {
return err
}
}
}
return nil
}
func (this *FileList) CleanAll() error {
if !this.isReady {
return nil
defer this.memoryCache.Clean()
for _, db := range this.dbList {
err := db.CleanAll()
if err != nil {
return err
}
}
this.memoryCache.Clean()
_, err := this.deleteAllStmt.Exec()
if err != nil {
return err
}
atomic.StoreInt64(&this.total, 0)
return nil
}
func (this *FileList) Stat(check func(hash string) bool) (*Stat, error) {
if !this.isReady {
return &Stat{}, nil
var result = &Stat{}
for _, db := range this.dbList {
if !db.IsReady() {
return &Stat{}, nil
}
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
_ = check
row := db.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
var stat = &Stat{}
err := row.Scan(&stat.Count, &stat.Size, &stat.ValueSize)
if err != nil {
return nil, err
}
result.Count += stat.Count
result.Size += stat.Size
result.ValueSize += stat.ValueSize
}
// 这里不设置过期时间、不使用 check 函数,目的是让查询更快速一些
row := this.statStmt.QueryRow()
if row.Err() != nil {
return nil, row.Err()
}
stat := &Stat{}
err := row.Scan(&stat.Count, &stat.Size, &stat.ValueSize)
if err != nil {
return nil, err
}
return stat, nil
return result, nil
}
// Count 总数量
@@ -449,9 +291,13 @@ func (this *FileList) Count() (int64, error) {
// IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return err
var db = this.getDB(hash)
if !db.IsReady() {
return nil
}
return db.IncreaseHit(hash)
}
// OnAdd 添加事件
@@ -465,162 +311,177 @@ func (this *FileList) OnRemove(f func(item *Item)) {
}
func (this *FileList) Close() error {
this.isClosed = true
this.isReady = false
this.memoryCache.Destroy()
if this.db != nil {
_ = this.existsByHashStmt.Close()
_ = this.insertStmt.Close()
_ = this.selectByHashStmt.Close()
_ = this.deleteByHashStmt.Close()
_ = this.statStmt.Close()
_ = this.purgeStmt.Close()
_ = this.deleteAllStmt.Close()
_ = this.insertHitStmt.Close()
_ = this.increaseHitStmt.Close()
_ = this.deleteHitByHashStmt.Close()
_ = this.lfuHitsStmt.Close()
return this.db.Close()
for _, db := range this.dbList {
if db != nil {
_ = db.Close()
}
}
return nil
}
// 初始化
func (this *FileList) initTables(db *sql.DB, times int) error {
// 检查是否存在
_, err := db.Exec(`SELECT id FROM "` + this.itemsTableName + `" LIMIT 1`)
var notFound = false
func (this *FileList) GetDBIndex(hash string) uint64 {
return fnv.HashString(hash) % CountFileDB
}
func (this *FileList) getDB(hash string) *FileListDB {
return this.dbList[fnv.HashString(hash)%CountFileDB]
}
func (this *FileList) remove(hash string) (notFound bool, err error) {
var db = this.getDB(hash)
if !db.IsReady() {
return false, nil
}
// 从缓存中删除
this.memoryCache.Delete(hash)
var row = db.selectByHashStmt.QueryRow(hash)
if row.Err() != nil {
if row.Err() == sql.ErrNoRows {
return true, nil
}
return false, row.Err()
}
var item = &Item{Type: ItemTypeFile}
err = row.Scan(&item.Key, &item.HeaderSize, &item.BodySize, &item.MetaSize, &item.ExpiredAt)
if err != nil {
notFound = true
}
{
// expiredAt - 过期时间,用来判断有无过期
// staleAt - 陈旧最大时间,用来清理缓存
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"key" varchar(1024),
"headerSize" integer DEFAULT 0,
"bodySize" integer DEFAULT 0,
"metaSize" integer DEFAULT 0,
"expiredAt" integer DEFAULT 0,
"staleAt" integer DEFAULT 0,
"createdAt" integer DEFAULT 0,
"host" varchar(128),
"serverId" integer
);
CREATE INDEX IF NOT EXISTS "createdAt"
ON "` + this.itemsTableName + `" (
"createdAt" ASC
);
CREATE INDEX IF NOT EXISTS "expiredAt"
ON "` + this.itemsTableName + `" (
"expiredAt" ASC
);
CREATE INDEX IF NOT EXISTS "staleAt"
ON "` + this.itemsTableName + `" (
"staleAt" ASC
);
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
ON "` + this.itemsTableName + `" (
"hash" ASC
);
CREATE INDEX IF NOT EXISTS "serverId"
ON "` + this.itemsTableName + `" (
"serverId" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := db.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(db, times+1)
}
return err
}
return err
if err == sql.ErrNoRows {
return true, nil
}
return false, err
}
// 如果数据为空,从老数据中加载数据
if notFound {
// v2 => v3
remotelogs.Println("CACHE", "transferring old data from v2 to v3 ...")
result, err := db.Exec(`INSERT INTO "` + this.itemsTableName + `" ("id", "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "createdAt", "host", "serverId", "staleAt") SELECT "id", "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "createdAt", "host", "serverId", "expiredAt"+600 FROM cacheItems_v2`)
if err == nil {
count, _ := result.RowsAffected()
remotelogs.Println("CACHE", "transfer old data from v2 to v3 finished, "+types.String(count)+" rows transferred")
}
_, err = db.deleteByHashStmt.Exec(hash)
if err != nil {
return false, err
}
{
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.hitsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"week1Hits" integer DEFAULT 0,
"week2Hits" integer DEFAULT 0,
"week" varchar(6)
);
atomic.AddInt64(&this.total, -1)
CREATE UNIQUE INDEX IF NOT EXISTS "hits_hash"
ON "` + this.hitsTableName + `" (
"hash" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := db.Exec(`DROP TABLE "` + this.hitsTableName + `"`)
if dropErr == nil {
return this.initTables(db, times+1)
}
return err
}
return err
}
_, err = db.deleteHitByHashStmt.Exec(hash)
if err != nil {
return false, err
}
return nil
if this.onRemove != nil {
this.onRemove(item)
}
return false, nil
}
// 删除过期不用的表格
func (this *FileList) removeOldTables() error {
rows, err := this.db.Query(`SELECT "name" FROM sqlite_master WHERE "type"='table'`)
// 升级老版本数据库
func (this *FileList) upgradeOldDB() {
if len(this.oldDir) == 0 {
return
}
_ = this.UpgradeV3(this.oldDir, false)
}
func (this *FileList) UpgradeV3(oldDir string, brokenOnError bool) error {
// index.db
var indexDBPath = oldDir + "/index.db"
_, err := os.Stat(indexDBPath)
if err != nil {
return nil
}
remotelogs.Println("CACHE", "upgrading local database from '"+oldDir+"' ...")
defer func() {
_ = os.Remove(indexDBPath)
remotelogs.Println("CACHE", "upgrading local database finished")
}()
db, err := sql.Open("sqlite3", "file:"+indexDBPath+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
defer func() {
_ = rows.Close()
_ = db.Close()
}()
for rows.Next() {
var name string
err = rows.Scan(&name)
var isFinished = false
var offset = 0
var count = 10000
for {
if isFinished {
break
}
err = func() error {
defer func() {
offset += count
}()
rows, err := db.Query(`SELECT "hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "createdAt", "host", "serverId" FROM "cacheItems_v3" ORDER BY "id" ASC LIMIT ?, ?`, offset, count)
if err != nil {
return err
}
defer func() {
_ = rows.Close()
}()
var hash = ""
var key = ""
var headerSize int64
var bodySize int64
var metaSize int64
var expiredAt int64
var staleAt int64
var createdAt int64
var host string
var serverId int64
isFinished = true
for rows.Next() {
isFinished = false
err = rows.Scan(&hash, &key, &headerSize, &bodySize, &metaSize, &expiredAt, &staleAt, &createdAt, &host, &serverId)
if err != nil {
if brokenOnError {
return err
}
return nil
}
err = this.Add(hash, &Item{
Type: ItemTypeFile,
Key: key,
ExpiredAt: expiredAt,
StaleAt: staleAt,
HeaderSize: headerSize,
BodySize: bodySize,
MetaSize: metaSize,
Host: host,
ServerId: serverId,
Week1Hits: 0,
Week2Hits: 0,
Week: 0,
})
if err != nil {
if brokenOnError {
return err
}
}
}
return nil
}()
if err != nil {
return err
}
if lists.ContainsString(this.oldTables, name) {
// 异步执行
goman.New(func() {
remotelogs.Println("CACHE", "remove old table '"+name+"' ...")
_, _ = this.db.Exec(`DROP TABLE "` + name + `"`)
remotelogs.Println("CACHE", "remove old table '"+name+"' done")
})
}
time.Sleep(1 * time.Second)
}
return nil
}

View File

@@ -0,0 +1,484 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"database/sql"
"errors"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/iwind/TeaGo/types"
timeutil "github.com/iwind/TeaGo/utils/time"
"runtime"
"strings"
"time"
)
type FileListDB struct {
readDB *dbs.DB
writeDB *dbs.DB
itemsTableName string
hitsTableName string
total int64
isClosed bool
isReady bool
// cacheItems
existsByHashStmt *dbs.Stmt // 根据hash检查是否存在
insertStmt *dbs.Stmt // 写入数据
selectByHashStmt *dbs.Stmt // 使用hash查询数据
deleteByHashStmt *dbs.Stmt // 根据hash删除数据
statStmt *dbs.Stmt // 统计
purgeStmt *dbs.Stmt // 清理
deleteAllStmt *dbs.Stmt // 删除所有数据
listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
// hits
insertHitStmt *dbs.Stmt // 写入数据
increaseHitStmt *dbs.Stmt // 增加点击量
deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
lfuHitsStmt *dbs.Stmt // 读取老的数据
}
func NewFileListDB() *FileListDB {
return &FileListDB{}
}
func (this *FileListDB) Open(dbPath string) error {
// write db
writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
if err != nil {
return errors.New("open write database failed: " + err.Error())
}
writeDB.SetMaxOpenConns(1)
// TODO 耗时过长,暂时不整理数据库
/**_, err = db.Exec("VACUUM")
if err != nil {
return err
}**/
this.writeDB = dbs.NewDB(writeDB)
if teaconst.EnableDBStat {
this.writeDB.EnableStat(true)
}
// read db
readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
if err != nil {
return errors.New("open read database failed: " + err.Error())
}
readDB.SetMaxOpenConns(runtime.NumCPU())
this.readDB = dbs.NewDB(readDB)
if teaconst.EnableDBStat {
this.readDB.EnableStat(true)
}
return nil
}
func (this *FileListDB) Init() error {
this.itemsTableName = "cacheItems"
this.hitsTableName = "hits"
// 创建
var err = this.initTables(1)
if err != nil {
return errors.New("init tables failed: " + err.Error())
}
// 读取总数量
row := this.readDB.QueryRow(`SELECT COUNT(*) FROM "` + this.itemsTableName + `"`)
if row.Err() != nil {
return row.Err()
}
var total int64
err = row.Scan(&total)
if err != nil {
return err
}
this.total = total
// 常用语句
this.existsByHashStmt, err = this.readDB.Prepare(`SELECT "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? AND expiredAt>? LIMIT 1`)
if err != nil {
return err
}
this.insertStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
this.selectByHashStmt, err = this.readDB.Prepare(`SELECT "key", "headerSize", "bodySize", "metaSize", "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? LIMIT 1`)
if err != nil {
return err
}
this.deleteByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.statStmt, err = this.readDB.Prepare(`SELECT COUNT(*), IFNULL(SUM(headerSize+bodySize+metaSize), 0), IFNULL(SUM(headerSize+bodySize), 0) FROM "` + this.itemsTableName + `"`)
if err != nil {
return err
}
this.purgeStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" WHERE staleAt<=? LIMIT ?`)
if err != nil {
return err
}
this.deleteAllStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.itemsTableName + `"`)
if err != nil {
return err
}
this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
if err != nil {
return err
}
this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
return err
}
this.isReady = true
return nil
}
func (this *FileListDB) IsReady() bool {
return this.isReady
}
func (this *FileListDB) Total() int64 {
return this.total
}
func (this *FileListDB) Add(hash string, item *Item) error {
if item.StaleAt == 0 {
item.StaleAt = item.ExpiredAt
}
// 放入队列
_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
if err != nil {
return err
}
return nil
}
func (this *FileListDB) ListExpiredItems(count int) (hashList []string, err error) {
if !this.isReady {
return nil, nil
}
if count <= 0 {
count = 100
}
rows, err := this.purgeStmt.Query(time.Now().Unix(), count)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
return nil, err
}
hashList = append(hashList, hash)
}
return hashList, nil
}
func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
if !this.isReady {
return nil, nil
}
if count <= 0 {
count = 100
}
hashList, err = this.listLFUItems(count)
if err != nil {
return
}
if len(hashList) > count/2 {
return
}
// 不足补齐
olderHashList, err := this.listOlderItems(count - len(hashList))
if err != nil {
return nil, err
}
hashList = append(hashList, olderHashList...)
return
}
func (this *FileListDB) IncreaseHit(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return err
}
func (this *FileListDB) CleanPrefix(prefix string) error {
if !this.isReady {
return nil
}
var count = int64(10000)
var staleLife = 600 // TODO 需要可以设置
var unixTime = utils.UnixTime() // 只删除当前的,不删除新的
for {
result, err := this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET expiredAt=0,staleAt=? WHERE id IN (SELECT id FROM "`+this.itemsTableName+`" WHERE expiredAt>0 AND createdAt<=? AND INSTR("key", ?)=1 LIMIT `+types.String(count)+`)`, unixTime+int64(staleLife), unixTime, prefix)
if err != nil {
return err
}
affectedRows, err := result.RowsAffected()
if err != nil {
return err
}
if affectedRows < count {
return nil
}
}
}
func (this *FileListDB) CleanAll() error {
if !this.isReady {
return nil
}
_, err := this.deleteAllStmt.Exec()
if err != nil {
return err
}
return nil
}
func (this *FileListDB) Close() error {
this.isClosed = true
this.isReady = false
if this.existsByHashStmt != nil {
_ = this.existsByHashStmt.Close()
}
if this.insertStmt != nil {
_ = this.insertStmt.Close()
}
if this.selectByHashStmt != nil {
_ = this.selectByHashStmt.Close()
}
if this.deleteByHashStmt != nil {
_ = this.deleteByHashStmt.Close()
}
if this.statStmt != nil {
_ = this.statStmt.Close()
}
if this.purgeStmt != nil {
_ = this.purgeStmt.Close()
}
if this.deleteAllStmt != nil {
_ = this.deleteAllStmt.Close()
}
if this.listOlderItemsStmt != nil {
_ = this.listOlderItemsStmt.Close()
}
if this.insertHitStmt != nil {
_ = this.insertHitStmt.Close()
}
if this.increaseHitStmt != nil {
_ = this.increaseHitStmt.Close()
}
if this.deleteHitByHashStmt != nil {
_ = this.deleteHitByHashStmt.Close()
}
if this.lfuHitsStmt != nil {
_ = this.lfuHitsStmt.Close()
}
var errStrings []string
if this.readDB != nil {
err := this.readDB.Close()
if err != nil {
errStrings = append(errStrings, err.Error())
}
}
if this.writeDB != nil {
err := this.writeDB.Close()
if err != nil {
errStrings = append(errStrings, err.Error())
}
}
if len(errStrings) == 0 {
return nil
}
return errors.New("close database failed: " + strings.Join(errStrings, ", "))
}
// 初始化
func (this *FileListDB) initTables(times int) error {
{
// expiredAt - 过期时间,用来判断有无过期
// staleAt - 陈旧最大时间,用来清理缓存
_, err := this.writeDB.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"key" varchar(1024),
"tag" varchar(64),
"headerSize" integer DEFAULT 0,
"bodySize" integer DEFAULT 0,
"metaSize" integer DEFAULT 0,
"expiredAt" integer DEFAULT 0,
"staleAt" integer DEFAULT 0,
"createdAt" integer DEFAULT 0,
"host" varchar(128),
"serverId" integer
);
CREATE INDEX IF NOT EXISTS "createdAt"
ON "` + this.itemsTableName + `" (
"createdAt" ASC
);
CREATE INDEX IF NOT EXISTS "expiredAt"
ON "` + this.itemsTableName + `" (
"expiredAt" ASC
);
CREATE INDEX IF NOT EXISTS "staleAt"
ON "` + this.itemsTableName + `" (
"staleAt" ASC
);
CREATE UNIQUE INDEX IF NOT EXISTS "hash"
ON "` + this.itemsTableName + `" (
"hash" ASC
);
CREATE INDEX IF NOT EXISTS "serverId"
ON "` + this.itemsTableName + `" (
"serverId" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(times + 1)
}
return err
}
return err
}
}
{
_, err := this.writeDB.Exec(`CREATE TABLE IF NOT EXISTS "` + this.hitsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"week1Hits" integer DEFAULT 0,
"week2Hits" integer DEFAULT 0,
"week" varchar(6)
);
CREATE UNIQUE INDEX IF NOT EXISTS "hits_hash"
ON "` + this.hitsTableName + `" (
"hash" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.hitsTableName + `"`)
if dropErr == nil {
return this.initTables(times + 1)
}
return err
}
return err
}
}
return nil
}
func (this *FileListDB) listLFUItems(count int) (hashList []string, err error) {
rows, err := this.lfuHitsStmt.Query(count)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
return nil, err
}
hashList = append(hashList, hash)
}
return hashList, nil
}
func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) {
rows, err := this.listOlderItemsStmt.Query(count)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
return nil, err
}
hashList = append(hashList, hash)
}
return hashList, nil
}

View File

@@ -1,8 +1,9 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
package caches_test
import (
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/rands"
@@ -15,23 +16,33 @@ import (
)
func TestFileList_Init(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
t.Log("ok")
}
func TestFileList_Add(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
err = list.Add(stringutil.Md5("123456"), &Item{
defer func() {
_ = list.Close()
}()
var hash = stringutil.Md5("123456")
t.Log("db index:", list.GetDBIndex(hash))
err = list.Add(hash, &caches.Item{
Key: "123456",
ExpiredAt: time.Now().Unix(),
ExpiredAt: time.Now().Unix() + 1,
HeaderSize: 1,
MetaSize: 2,
BodySize: 3,
@@ -41,19 +52,27 @@ func TestFileList_Add(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Log(list.Exist(hash))
t.Log("ok")
}
func TestFileList_Add_Many(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
before := time.Now()
for i := 0; i < 2000_0000; i++ {
for i := 0; i < 100_000; i++ {
u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
_ = list.Add(stringutil.Md5(u), &Item{
_ = list.Add(stringutil.Md5(u), &caches.Item{
Key: u,
ExpiredAt: time.Now().Unix() + 3600,
HeaderSize: 1,
@@ -72,36 +91,46 @@ func TestFileList_Add_Many(t *testing.T) {
}
func TestFileList_Exist(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
before := time.Now()
defer func() {
_ = list.Close()
}()
total, _ := list.Count()
t.Log("total:", total)
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
{
exists, err := list.Exist(stringutil.Md5("123456"))
var hash = stringutil.Md5("123456")
exists, err := list.Exist(hash)
if err != nil {
t.Fatal(err)
}
t.Log("exists:", exists)
t.Log(hash, "exists:", exists)
}
{
exists, err := list.Exist(stringutil.Md5("http://edge.teaos.cn/1234561"))
var hash = stringutil.Md5("http://edge.teaos.cn/1234561")
exists, err := list.Exist(hash)
if err != nil {
t.Fatal(err)
}
t.Log("exists:", exists)
t.Log(hash, "exists:", exists)
}
}
func TestFileList_Exist_Many_DB(t *testing.T) {
// 测试在多个数据库下的性能
var listSlice = []ListInterface{}
var listSlice = []caches.ListInterface{}
for i := 1; i <= 10; i++ {
list := NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
list := caches.NewFileList(Tea.Root + "/data/data" + strconv.Itoa(i))
err := list.Init()
if err != nil {
t.Fatal(err)
@@ -151,13 +180,18 @@ func TestFileList_Exist_Many_DB(t *testing.T) {
}
func TestFileList_CleanPrefix(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
before := time.Now()
err = list.CleanPrefix("1234")
err = list.CleanPrefix("123")
if err != nil {
t.Fatal(err)
}
@@ -165,12 +199,17 @@ func TestFileList_CleanPrefix(t *testing.T) {
}
func TestFileList_Remove(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
list.OnRemove(func(item *Item) {
defer func() {
_ = list.Close()
}()
list.OnRemove(func(item *caches.Item) {
t.Logf("remove %#v", item)
})
err = list.Remove(stringutil.Md5("123456"))
@@ -178,30 +217,71 @@ func TestFileList_Remove(t *testing.T) {
t.Fatal(err)
}
t.Log("ok")
t.Log("===count===")
t.Log(list.Count())
}
func TestFileList_Purge(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
_, err = list.Purge(2, func(hash string) error {
defer func() {
_ = list.Close()
}()
var count = 0
_, err = list.Purge(caches.CountFileDB*2, func(hash string) error {
t.Log(hash)
count++
return nil
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
t.Log("ok, purged", count, "items")
}
func TestFileList_Stat(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
func TestFileList_PurgeLFU(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
err = list.IncreaseHit(stringutil.Md5("123456"))
if err != nil {
t.Fatal(err)
}
var count = 0
err = list.PurgeLFU(caches.CountFileDB*2, func(hash string) error {
t.Log(hash)
count++
return nil
})
if err != nil {
t.Fatal(err)
}
t.Log("ok, purged", count, "items")
}
func TestFileList_Stat(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
stat, err := list.Stat(nil)
if err != nil {
t.Fatal(err)
@@ -210,7 +290,7 @@ func TestFileList_Stat(t *testing.T) {
}
func TestFileList_Count(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
list := caches.NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
@@ -225,7 +305,7 @@ func TestFileList_Count(t *testing.T) {
}
func TestFileList_CleanAll(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
list := caches.NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
@@ -238,58 +318,17 @@ func TestFileList_CleanAll(t *testing.T) {
t.Log(list.Count())
}
func TestFileList_Conflict(t *testing.T) {
list := NewFileList(Tea.Root + "/data").(*FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
rows, err := list.purgeStmt.Query(time.Now().Unix(), 1000)
if err != nil {
t.Fatal(err)
}
go func() {
time.Sleep(5 * time.Second)
_ = rows.Close()
}()
t.Log("before exists")
t.Log(list.Exist("123456"))
t.Log("after exists")
}
func TestFileList_IIF(t *testing.T) {
list := NewFileList(Tea.Root + "/data").(*FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
rows, err := list.db.Query("SELECT IIF(0, 2, 3)")
if err != nil {
t.Fatal(err)
}
defer func() {
_ = rows.Close()
}()
if rows.Next() {
var result int
err = rows.Scan(&result)
if err != nil {
t.Fatal(err)
}
t.Log("result:", result)
}
}
func TestFileList_IncreaseHit(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
@@ -303,13 +342,32 @@ func TestFileList_IncreaseHit(t *testing.T) {
t.Log("ok")
}
func TestFileList_UpgradeV3(t *testing.T) {
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p43").(*caches.FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
defer func() {
_ = list.Close()
}()
err = list.UpgradeV3("/Users/WorkSpace/EdgeProject/EdgeCache/p43", false)
if err != nil {
t.Log(err)
return
}
t.Log("ok")
}
func BenchmarkFileList_Exist(b *testing.B) {
list := NewFileList(Tea.Root + "/data")
var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
err := list.Init()
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f")
_, _ = list.Exist("f0eb5b87e0b0041f3917002c0707475f" + types.String(i))
}
}

View File

@@ -25,6 +25,7 @@ func init() {
type Manager struct {
// 全局配置
MaxDiskCapacity *shared.SizeCapacity
DiskDir string
MaxMemoryCapacity *shared.SizeCapacity
policyMap map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
@@ -49,6 +50,11 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
newPolicyIds := []int64{}
for _, policy := range newPolicies {
// 使用节点单独的缓存目录
if len(this.DiskDir) > 0 {
policy.UpdateDiskDir(this.DiskDir)
}
newPolicyIds = append(newPolicyIds, policy.Id)
}
@@ -99,7 +105,19 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
} else {
// 检查policy是否有变化
if !storage.Policy().IsSame(policy) {
remotelogs.Println("CACHE", "policy "+strconv.FormatInt(policy.Id, 10)+" changed")
// 检查是否可以直接修改
if storage.CanUpdatePolicy(policy) {
err := policy.Init()
if err != nil {
remotelogs.Error("CACHE", "reload policy '"+types.String(policy.Id)+"' failed: init policy failed: "+err.Error())
continue
}
remotelogs.Println("CACHE", "reload policy '"+types.String(policy.Id)+"'")
storage.UpdatePolicy(policy)
continue
}
remotelogs.Println("CACHE", "restart policy '"+types.String(policy.Id)+"'")
// 停止老的
storage.Stop()

View File

@@ -2,19 +2,20 @@ package caches
import (
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
"github.com/iwind/TeaGo/Tea"
"testing"
)
func TestManager_UpdatePolicies(t *testing.T) {
{
policies := []*serverconfigs.HTTPCachePolicy{}
var policies = []*serverconfigs.HTTPCachePolicy{}
SharedManager.UpdatePolicies(policies)
printManager(t)
}
{
policies := []*serverconfigs.HTTPCachePolicy{
var policies = []*serverconfigs.HTTPCachePolicy{
{
Id: 1,
Type: serverconfigs.CachePolicyStorageFile,
@@ -42,7 +43,7 @@ func TestManager_UpdatePolicies(t *testing.T) {
}
{
policies := []*serverconfigs.HTTPCachePolicy{
var policies = []*serverconfigs.HTTPCachePolicy{
{
Id: 1,
Type: serverconfigs.CachePolicyStorageFile,
@@ -71,6 +72,50 @@ func TestManager_UpdatePolicies(t *testing.T) {
}
}
func TestManager_ChangePolicy_Memory(t *testing.T) {
var policies = []*serverconfigs.HTTPCachePolicy{
{
Id: 1,
Type: serverconfigs.CachePolicyStorageMemory,
Options: map[string]interface{}{},
Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
},
}
SharedManager.UpdatePolicies(policies)
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
{
Id: 1,
Type: serverconfigs.CachePolicyStorageMemory,
Options: map[string]interface{}{},
Capacity: &shared.SizeCapacity{Count: 2, Unit: shared.SizeCapacityUnitGB},
},
})
}
func TestManager_ChangePolicy_File(t *testing.T) {
var policies = []*serverconfigs.HTTPCachePolicy{
{
Id: 1,
Type: serverconfigs.CachePolicyStorageFile,
Options: map[string]interface{}{
"dir": Tea.Root + "/data/cache-index/p1",
},
Capacity: &shared.SizeCapacity{Count: 1, Unit: shared.SizeCapacityUnitGB},
},
}
SharedManager.UpdatePolicies(policies)
SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{
{
Id: 1,
Type: serverconfigs.CachePolicyStorageFile,
Options: map[string]interface{}{
"dir": Tea.Root + "/data/cache-index/p1",
},
Capacity: &shared.SizeCapacity{Count: 2, Unit: shared.SizeCapacityUnitGB},
},
})
}
func printManager(t *testing.T) {
t.Log("===manager==")
t.Log("storage:")

View File

@@ -4,28 +4,37 @@ package caches
import (
"encoding/json"
"errors"
"io/ioutil"
)
// PartialRanges 内容分区范围定义
type PartialRanges struct {
ranges [][2]int64
Ranges [][2]int64 `json:"ranges"`
}
// NewPartialRanges 获取新对象
func NewPartialRanges() *PartialRanges {
return &PartialRanges{ranges: [][2]int64{}}
return &PartialRanges{Ranges: [][2]int64{}}
}
// NewPartialRangesFromJSON 从JSON中解析范围
func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
var rs = [][2]int64{}
var rs = NewPartialRanges()
err := json.Unmarshal(data, &rs)
if err != nil {
return nil, err
}
var r = NewPartialRanges()
r.ranges = rs
return r, nil
return rs, nil
}
func NewPartialRangesFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return NewPartialRangesFromJSON(data)
}
// Add 添加新范围
@@ -36,46 +45,41 @@ func (this *PartialRanges) Add(begin int64, end int64) {
var nr = [2]int64{begin, end}
var count = len(this.ranges)
var count = len(this.Ranges)
if count == 0 {
this.ranges = [][2]int64{nr}
this.Ranges = [][2]int64{nr}
return
}
// insert
// TODO 将来使用二分法改进
var index = -1
for i, r := range this.ranges {
for i, r := range this.Ranges {
if r[0] > begin || (r[0] == begin && r[1] >= end) {
index = i
this.ranges = append(this.ranges, [2]int64{})
copy(this.ranges[index+1:], this.ranges[index:])
this.ranges[index] = nr
this.Ranges = append(this.Ranges, [2]int64{})
copy(this.Ranges[index+1:], this.Ranges[index:])
this.Ranges[index] = nr
break
}
}
if index == -1 {
index = count
this.ranges = append(this.ranges, nr)
this.Ranges = append(this.Ranges, nr)
}
this.merge(index)
}
// Ranges 获取所有范围
func (this *PartialRanges) Ranges() [][2]int64 {
return this.ranges
}
// Contains 检查是否包含某个范围
func (this *PartialRanges) Contains(begin int64, end int64) bool {
if len(this.ranges) == 0 {
return true
if len(this.Ranges) == 0 {
return false
}
// TODO 使用二分法查找改进性能
for _, r2 := range this.ranges {
for _, r2 := range this.Ranges {
if r2[0] <= begin && r2[1] >= end {
return true
}
@@ -84,23 +88,65 @@ func (this *PartialRanges) Contains(begin int64, end int64) bool {
return false
}
// Nearest 查找最近的某个范围
func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool) {
if len(this.Ranges) == 0 {
return
}
// TODO 使用二分法查找改进性能
for _, r2 := range this.Ranges {
if r2[0] <= begin && r2[1] > begin {
r = [2]int64{begin, this.min(end, r2[1])}
ok = true
return
}
}
return
}
// AsJSON 转换为JSON
func (this *PartialRanges) AsJSON() ([]byte, error) {
return json.Marshal(this.ranges)
return json.Marshal(this)
}
// WriteToFile 写入到文件中
func (this *PartialRanges) WriteToFile(path string) error {
data, err := this.AsJSON()
if err != nil {
return errors.New("convert to json failed: " + err.Error())
}
return ioutil.WriteFile(path, data, 0666)
}
// ReadFromFile 从文件中读取
func (this *PartialRanges) ReadFromFile(path string) (*PartialRanges, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return NewPartialRangesFromJSON(data)
}
func (this *PartialRanges) Max() int64 {
if len(this.Ranges) > 0 {
return this.Ranges[len(this.Ranges)-1][1]
}
return 0
}
func (this *PartialRanges) merge(index int) {
// forward
var lastIndex = index
for i := index; i >= 1; i-- {
var curr = this.ranges[i]
var prev = this.ranges[i-1]
var curr = this.Ranges[i]
var prev = this.Ranges[i-1]
var w1 = this.w(curr)
var w2 = this.w(prev)
if w1+w2 >= this.max(curr[1], prev[1])-this.min(curr[0], prev[0])-1 {
prev = [2]int64{this.min(curr[0], prev[0]), this.max(curr[1], prev[1])}
this.ranges[i-1] = prev
this.ranges = append(this.ranges[:i], this.ranges[i+1:]...)
this.Ranges[i-1] = prev
this.Ranges = append(this.Ranges[:i], this.Ranges[i+1:]...)
lastIndex = i - 1
} else {
break
@@ -109,15 +155,15 @@ func (this *PartialRanges) merge(index int) {
// backward
index = lastIndex
for index < len(this.ranges)-1 {
var curr = this.ranges[index]
var next = this.ranges[index+1]
for index < len(this.Ranges)-1 {
var curr = this.Ranges[index]
var next = this.Ranges[index+1]
var w1 = this.w(curr)
var w2 = this.w(next)
if w1+w2 >= this.max(curr[1], next[1])-this.min(curr[0], next[0])-1 {
curr = [2]int64{this.min(curr[0], next[0]), this.max(curr[1], next[1])}
this.ranges = append(this.ranges[:index], this.ranges[index+1:]...)
this.ranges[index] = curr
this.Ranges = append(this.Ranges[:index], this.Ranges[index+1:]...)
this.Ranges[index] = curr
} else {
break
}

View File

@@ -21,7 +21,8 @@ func TestNewPartialRanges(t *testing.T) {
r.Add(200, 1000)
r.Add(200, 10040)
logs.PrintAsJSON(r.Ranges())
logs.PrintAsJSON(r.Ranges, t)
t.Log("max:", r.Max())
}
func TestNewPartialRanges1(t *testing.T) {
@@ -35,7 +36,7 @@ func TestNewPartialRanges1(t *testing.T) {
r.Add(200, 300)
r.Add(1, 1000)
var rs = r.Ranges()
var rs = r.Ranges
logs.PrintAsJSON(rs, t)
a.IsTrue(len(rs) == 1)
if len(rs) == 1 {
@@ -56,7 +57,7 @@ func TestNewPartialRanges2(t *testing.T) {
r.Add(303, 304)
r.Add(250, 400)
var rs = r.Ranges()
var rs = r.Ranges
logs.PrintAsJSON(rs, t)
}
@@ -68,7 +69,7 @@ func TestNewPartialRanges3(t *testing.T) {
r.Add(200, 300)
r.Add(250, 400)
var rs = r.Ranges()
var rs = r.Ranges
logs.PrintAsJSON(rs, t)
}
@@ -83,7 +84,7 @@ func TestNewPartialRanges4(t *testing.T) {
r.Add(410, 415)
r.Add(400, 409)
var rs = r.Ranges()
var rs = r.Ranges
logs.PrintAsJSON(rs, t)
t.Log(r.Contains(400, 416))
}
@@ -93,7 +94,57 @@ func TestNewPartialRanges5(t *testing.T) {
for j := 0; j < 1000; j++ {
r.Add(int64(j), int64(j+100))
}
logs.PrintAsJSON(r.Ranges(), t)
logs.PrintAsJSON(r.Ranges, t)
}
func TestNewPartialRanges_Nearest(t *testing.T) {
{
// nearby
var r = caches.NewPartialRanges()
r.Add(301, 400)
r.Add(401, 500)
r.Add(501, 600)
t.Log(r.Nearest(100, 200))
t.Log(r.Nearest(300, 350))
t.Log(r.Nearest(302, 350))
}
{
// nearby
var r = caches.NewPartialRanges()
r.Add(301, 400)
r.Add(450, 500)
r.Add(550, 600)
t.Log(r.Nearest(100, 200))
t.Log(r.Nearest(300, 350))
t.Log(r.Nearest(302, 350))
t.Log(r.Nearest(302, 440))
t.Log(r.Nearest(302, 1000))
}
}
func TestNewPartialRanges_Large_Range(t *testing.T) {
var a = assert.NewAssertion(t)
var largeSize int64 = 10000000000000
t.Log(largeSize/1024/1024/1024, "G")
var r = caches.NewPartialRanges()
r.Add(1, largeSize)
jsonData, err := r.AsJSON()
if err != nil {
t.Fatal(err)
}
t.Log(string(jsonData))
r2, err := caches.NewPartialRangesFromJSON(jsonData)
if err != nil {
t.Fatal(err)
}
a.IsTrue(largeSize == r2.Ranges[0][1])
}
func TestNewPartialRanges_AsJSON(t *testing.T) {
@@ -111,7 +162,7 @@ func TestNewPartialRanges_AsJSON(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Log(r2.Ranges())
t.Log(r2.Ranges)
}
func BenchmarkNewPartialRanges(b *testing.B) {

View File

@@ -1,5 +1,7 @@
package caches
import "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
type ReaderFunc func(n int) (goNext bool, err error)
type Reader interface {
@@ -36,6 +38,9 @@ type Reader interface {
// BodySize Body Size
BodySize() int64
// ContainsRange 是否包含某个区间内容
ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool)
// Close 关闭
Close() error
}

View File

@@ -3,6 +3,7 @@ package caches
import (
"encoding/binary"
"errors"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
"io"
"os"
@@ -332,6 +333,11 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
return nil
}
// ContainsRange 是否包含某些区间内容
func (this *FileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
return r, true
}
func (this *FileReader) Close() error {
if this.openFileCache != nil {
if this.isClosed {

View File

@@ -2,6 +2,7 @@ package caches
import (
"errors"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"io"
)
@@ -197,6 +198,11 @@ func (this *MemoryReader) ReadBodyRange(buf []byte, start int64, end int64, call
return nil
}
// ContainsRange 是否包含某些区间内容
func (this *MemoryReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
return r, true
}
func (this *MemoryReader) Close() error {
return nil
}

View File

@@ -0,0 +1,144 @@
package caches
import (
"encoding/binary"
"errors"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
"io"
"os"
)
type PartialFileReader struct {
*FileReader
ranges *PartialRanges
rangePath string
}
func NewPartialFileReader(fp *os.File) *PartialFileReader {
return &PartialFileReader{
FileReader: NewFileReader(fp),
rangePath: partialRangesFilePath(fp.Name()),
}
}
func (this *PartialFileReader) Init() error {
return this.InitAutoDiscard(true)
}
func (this *PartialFileReader) InitAutoDiscard(autoDiscard bool) error {
if this.openFile != nil {
this.meta = this.openFile.meta
this.header = this.openFile.header
}
isOk := false
if autoDiscard {
defer func() {
if !isOk {
_ = this.discard()
}
}()
}
// 读取Range
ranges, err := NewPartialRangesFromFile(this.rangePath)
if err != nil {
return errors.New("read ranges failed: " + err.Error())
}
this.ranges = ranges
var buf = this.meta
if len(buf) == 0 {
buf = make([]byte, SizeMeta)
ok, err := this.readToBuff(this.fp, buf)
if err != nil {
return err
}
if !ok {
return ErrNotFound
}
this.meta = buf
}
this.expiresAt = int64(binary.BigEndian.Uint32(buf[:SizeExpiresAt]))
status := types.Int(string(buf[SizeExpiresAt : SizeExpiresAt+SizeStatus]))
if status < 100 || status > 999 {
return errors.New("invalid status")
}
this.status = status
// URL
urlLength := binary.BigEndian.Uint32(buf[SizeExpiresAt+SizeStatus : SizeExpiresAt+SizeStatus+SizeURLLength])
// header
headerSize := int(binary.BigEndian.Uint32(buf[SizeExpiresAt+SizeStatus+SizeURLLength : SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength]))
if headerSize == 0 {
return nil
}
this.headerSize = headerSize
this.headerOffset = int64(SizeMeta) + int64(urlLength)
// body
this.bodyOffset = this.headerOffset + int64(headerSize)
bodySize := int(binary.BigEndian.Uint64(buf[SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength : SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength+SizeBodyLength]))
if bodySize == 0 {
isOk = true
return nil
}
this.bodySize = int64(bodySize)
// read header
if this.openFileCache != nil && len(this.header) == 0 {
if headerSize > 0 && headerSize <= 512 {
this.header = make([]byte, headerSize)
_, err := this.fp.Seek(this.headerOffset, io.SeekStart)
if err != nil {
return err
}
_, err = this.readToBuff(this.fp, this.header)
if err != nil {
return err
}
}
}
isOk = true
return nil
}
// ContainsRange 是否包含某些区间内容
// 这里的 r 是已经经过格式化的
func (this *PartialFileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool) {
r2, ok = this.ranges.Nearest(r.Start(), r.End())
if ok && this.bodySize > 0 {
// 考虑可配置
var span int64 = 512 * 1024
if this.bodySize > 1<<30 {
span = 1 << 20
}
// 这里限制返回的最小缓存,防止因为返回的内容过小而导致请求过多
if r2.Length() < r.Length() && r2.Length() < span {
ok = false
}
}
return
}
// MaxLength 获取区间最大长度
func (this *PartialFileReader) MaxLength() int64 {
if this.bodySize > 0 {
return this.bodySize
}
return this.ranges.Max() + 1
}
func (this *PartialFileReader) discard() error {
_ = os.Remove(this.rangePath)
return this.FileReader.discard()
}

View File

@@ -1,20 +1,23 @@
package caches
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
"github.com/TeaOSLab/EdgeNode/internal/utils/sizes"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
@@ -44,7 +47,9 @@ const (
)
const (
HotItemSize = 1024
HotItemSize = 1024 // 热点数据数量
HotItemLifeSeconds int64 = 3600 // 热点数据生命周期
FileToMemoryMaxSize int64 = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
)
var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
@@ -55,7 +60,7 @@ var sharedWritingFileKeyLocker = sync.Mutex{}
// [expires time] | [ status ] | [url length] | [header length] | [body length] | [url] [header data] [body data]
type FileStorage struct {
policy *serverconfigs.HTTPCachePolicy
cacheConfig *serverconfigs.HTTPFileCacheStorage // 二级缓存
options *serverconfigs.HTTPFileCacheStorage // 二级缓存
memoryStorage *MemoryStorage // 一级缓存
totalSize int64
@@ -68,6 +73,8 @@ type FileStorage struct {
lastHotSize int
hotTicker *utils.Ticker
ignoreKeys *setutils.FixedSet
openFileCache *OpenFileCache
}
@@ -76,6 +83,7 @@ func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
policy: policy,
hotMap: map[string]*HotItem{},
lastHotSize: -1,
ignoreKeys: setutils.NewFixedSet(32768),
}
}
@@ -84,41 +92,125 @@ func (this *FileStorage) Policy() *serverconfigs.HTTPCachePolicy {
return this.policy
}
// CanUpdatePolicy 检查策略是否可以更新
func (this *FileStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool {
// 检查路径是否有变化
oldOptionsJSON, err := json.Marshal(this.policy.Options)
if err != nil {
return false
}
var oldOptions = &serverconfigs.HTTPFileCacheStorage{}
err = json.Unmarshal(oldOptionsJSON, oldOptions)
if err != nil {
return false
}
newOptionsJSON, err := json.Marshal(newPolicy.Options)
if err != nil {
return false
}
var newOptions = &serverconfigs.HTTPFileCacheStorage{}
err = json.Unmarshal(newOptionsJSON, newOptions)
if err != nil {
return false
}
if oldOptions.Dir == newOptions.Dir {
return true
}
return false
}
// UpdatePolicy 修改策略
func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) {
var oldOpenFileCache = this.options.OpenFileCache
this.policy = newPolicy
newOptionsJSON, err := json.Marshal(newPolicy.Options)
if err != nil {
return
}
var newOptions = &serverconfigs.HTTPFileCacheStorage{}
err = json.Unmarshal(newOptionsJSON, newOptions)
if err != nil {
remotelogs.Error("CACHE", "update policy '"+types.String(this.policy.Id)+"' failed: decode options failed: "+err.Error())
return
}
err = newOptions.Init()
if err != nil {
remotelogs.Error("CACHE", "update policy '"+types.String(this.policy.Id)+"' failed: init options failed: "+err.Error())
return
}
this.options = newOptions
var memoryStorage = this.memoryStorage
if memoryStorage != nil {
if newOptions.MemoryPolicy != nil && newOptions.MemoryPolicy.CapacityBytes() > 0 {
memoryStorage.UpdatePolicy(newOptions.MemoryPolicy)
} else {
memoryStorage.Stop()
this.memoryStorage = nil
}
} else if newOptions.MemoryPolicy != nil && this.options.MemoryPolicy.Capacity != nil && this.options.MemoryPolicy.Capacity.Count > 0 {
err = this.createMemoryStorage()
if err != nil {
remotelogs.Error("CACHE", "update policy '"+types.String(this.policy.Id)+"' failed: create memory storage failed: "+err.Error())
}
}
// open cache
oldOpenFileCacheJSON, _ := json.Marshal(oldOpenFileCache)
newOpenFileCacheJSON, _ := json.Marshal(this.options.OpenFileCache)
if bytes.Compare(oldOpenFileCacheJSON, newOpenFileCacheJSON) != 0 {
this.initOpenFileCache()
}
// Purge Ticker
if newPolicy.PersistenceAutoPurgeInterval != this.policy.PersistenceAutoPurgeInterval {
this.initPurgeTicker()
}
}
// Init 初始化
func (this *FileStorage) Init() error {
this.locker.Lock()
defer this.locker.Unlock()
before := time.Now()
var before = time.Now()
// 配置
cacheConfig := &serverconfigs.HTTPFileCacheStorage{}
var options = &serverconfigs.HTTPFileCacheStorage{}
optionsJSON, err := json.Marshal(this.policy.Options)
if err != nil {
return err
}
err = json.Unmarshal(optionsJSON, cacheConfig)
err = json.Unmarshal(optionsJSON, options)
if err != nil {
return err
}
this.cacheConfig = cacheConfig
this.options = options
if !filepath.IsAbs(this.cacheConfig.Dir) {
this.cacheConfig.Dir = Tea.Root + Tea.DS + this.cacheConfig.Dir
if !filepath.IsAbs(this.options.Dir) {
this.options.Dir = Tea.Root + Tea.DS + this.options.Dir
}
this.cacheConfig.Dir = filepath.Clean(this.cacheConfig.Dir)
var dir = this.cacheConfig.Dir
this.options.Dir = filepath.Clean(this.options.Dir)
var dir = this.options.Dir
if len(dir) == 0 {
return errors.New("[CACHE]cache storage dir can not be empty")
}
list := NewFileList(dir + "/p" + strconv.FormatInt(this.policy.Id, 10))
var list = NewFileList(dir + "/p" + types.String(this.policy.Id) + "/.indexes")
err = list.Init()
if err != nil {
return err
}
list.(*FileList).SetOldDir(dir + "/p" + types.String(this.policy.Id))
this.list = list
stat, err := list.Stat(func(hash string) bool {
return true
@@ -149,8 +241,8 @@ func (this *FileStorage) Init() error {
defer func() {
// 统计
count := stat.Count
size := stat.Size
var count = stat.Count
var size = stat.Size
cost := time.Since(before).Seconds() * 1000
sizeMB := strconv.FormatInt(size, 10) + " Bytes"
@@ -161,7 +253,7 @@ func (this *FileStorage) Init() error {
} else if size > 1024 {
sizeMB = fmt.Sprintf("%.3f K", float64(size)/1024)
}
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+this.cacheConfig.Dir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
remotelogs.Println("CACHE", "init policy "+strconv.FormatInt(this.policy.Id, 10)+" from '"+this.options.Dir+"', cost: "+fmt.Sprintf("%.2f", cost)+" ms, count: "+message.NewPrinter(language.English).Sprintf("%d", count)+", size: "+sizeMB)
}()
// 初始化list
@@ -171,64 +263,38 @@ func (this *FileStorage) Init() error {
}
// 加载内存缓存
if this.cacheConfig.MemoryPolicy != nil {
if this.cacheConfig.MemoryPolicy.Capacity != nil && this.cacheConfig.MemoryPolicy.Capacity.Count > 0 {
memoryPolicy := &serverconfigs.HTTPCachePolicy{
Id: this.policy.Id,
IsOn: this.policy.IsOn,
Name: this.policy.Name,
Description: this.policy.Description,
Capacity: this.cacheConfig.MemoryPolicy.Capacity,
MaxKeys: this.policy.MaxKeys,
MaxSize: &shared.SizeCapacity{Count: 128, Unit: shared.SizeCapacityUnitMB}, // TODO 将来可以修改
Type: serverconfigs.CachePolicyStorageMemory,
Options: this.policy.Options,
Life: this.policy.Life,
MinLife: this.policy.MinLife,
MaxLife: this.policy.MaxLife,
MemoryAutoPurgeCount: this.policy.MemoryAutoPurgeCount,
MemoryAutoPurgeInterval: this.policy.MemoryAutoPurgeInterval,
MemoryLFUFreePercent: this.policy.MemoryLFUFreePercent,
}
err = memoryPolicy.Init()
if err != nil {
return err
}
memoryStorage := NewMemoryStorage(memoryPolicy, this)
err = memoryStorage.Init()
if err != nil {
return err
}
this.memoryStorage = memoryStorage
if this.options.MemoryPolicy != nil && this.options.MemoryPolicy.Capacity != nil && this.options.MemoryPolicy.Capacity.Count > 0 {
err = this.createMemoryStorage()
if err != nil {
return err
}
}
// open file cache
if this.cacheConfig.OpenFileCache != nil && this.cacheConfig.OpenFileCache.IsOn && this.cacheConfig.OpenFileCache.Max > 0 {
this.openFileCache, err = NewOpenFileCache(this.cacheConfig.OpenFileCache.Max)
logs.Println("start open file cache")
if err != nil {
remotelogs.Error("CACHE", "open file cache failed: "+err.Error())
}
}
this.initOpenFileCache()
return nil
}
func (this *FileStorage) OpenReader(key string, useStale bool) (Reader, error) {
return this.openReader(key, true, useStale)
func (this *FileStorage) OpenReader(key string, useStale bool, isPartial bool) (Reader, error) {
return this.openReader(key, true, useStale, isPartial)
}
func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool) (Reader, error) {
func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool, isPartial bool) (Reader, error) {
// 使用陈旧缓存的时候,我们认为是短暂的,只需要从文件里检查即可
if useStale {
allowMemory = false
}
// 区间缓存只存在文件中
if isPartial {
allowMemory = false
}
// 先尝试内存缓存
if allowMemory && this.memoryStorage != nil {
reader, err := this.memoryStorage.OpenReader(key, useStale)
var memoryStorage = this.memoryStorage
if allowMemory && memoryStorage != nil {
reader, err := memoryStorage.OpenReader(key, useStale, isPartial)
if err == nil {
return reader, err
}
@@ -250,8 +316,9 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
// TODO 尝试使用mmap加快读取速度
var isOk = false
var openFile *OpenFile
if this.openFileCache != nil {
openFile = this.openFileCache.Get(path)
var openFileCache = this.openFileCache // 因为中间可能有修改,所以先赋值再获取
if openFileCache != nil {
openFile = openFileCache.Get(path)
}
var fp *os.File
var err error
@@ -269,13 +336,22 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
defer func() {
if !isOk {
_ = fp.Close()
_ = os.Remove(path)
_ = this.removeCacheFile(path)
}
}()
var reader = NewFileReader(fp)
reader.openFile = openFile
reader.openFileCache = this.openFileCache
var reader Reader
if isPartial {
var partialFileReader = NewPartialFileReader(fp)
partialFileReader.openFile = openFile
partialFileReader.openFileCache = openFileCache
reader = partialFileReader
} else {
var fileReader = NewFileReader(fp)
fileReader.openFile = openFile
fileReader.openFileCache = openFileCache
reader = fileReader
}
err = reader.Init()
if err != nil {
return nil, err
@@ -283,41 +359,8 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
// 增加点击量
// 1/1000采样
if allowMemory {
var rate = this.policy.PersistenceHitSampleRate
if rate <= 0 {
rate = 1000
}
if this.lastHotSize == 0 {
// 自动降低采样率来增加热点数据的缓存几率
rate = rate / 10
}
if rands.Int(0, rate) == 0 {
var hitErr = this.list.IncreaseHit(hash)
if hitErr != nil {
// 此错误可以忽略
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
}
// 增加到热点
// 这里不收录缓存尺寸过大的文件
if this.memoryStorage != nil && reader.BodySize() > 0 && reader.BodySize() < 128*1024*1024 {
this.hotMapLocker.Lock()
hotItem, ok := this.hotMap[key]
if ok {
hotItem.Hits++
hotItem.ExpiresAt = reader.expiresAt
} else if len(this.hotMap) < HotItemSize { // 控制数量
this.hotMap[key] = &HotItem{
Key: key,
ExpiresAt: reader.ExpiresAt(),
Status: reader.Status(),
Hits: 1,
}
}
this.hotMapLocker.Unlock()
}
}
if !isPartial && allowMemory && reader.BodySize() < FileToMemoryMaxSize {
this.increaseHit(key, hash, reader)
}
isOk = true
@@ -325,14 +368,34 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool)
}
// OpenWriter 打开缓存文件等待写入
func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, size int64, isPartial bool) (Writer, error) {
func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error) {
// 是否正在退出
if teaconst.IsQuiting {
return nil, ErrWritingUnavailable
}
// 是否已忽略
if this.ignoreKeys.Has(key) {
return nil, ErrEntityTooLarge
}
// 先尝试内存缓存
// 我们限定仅小文件优先存在内存中
if !isPartial && this.memoryStorage != nil && size > 0 && size < 32*1024*1024 {
writer, err := this.memoryStorage.OpenWriter(key, expiredAt, status, size, false)
var maxMemorySize = FileToMemoryMaxSize
if maxSize > maxMemorySize {
maxMemorySize = maxSize
}
var memoryStorage = this.memoryStorage
if !isPartial && memoryStorage != nil && ((size > 0 && size < maxMemorySize) || size < 0) {
writer, err := memoryStorage.OpenWriter(key, expiredAt, status, size, maxMemorySize, false)
if err == nil {
return writer, nil
}
// 如果队列满了,则等待
if err == ErrWritingQueueFull {
return nil, err
}
}
// 是否正在写入
@@ -367,7 +430,7 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
}
var hash = stringutil.Md5(key)
var dir = this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
var dir = this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
_, err = os.Stat(dir)
if err != nil {
if !os.IsNotExist(err) {
@@ -380,7 +443,8 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
}
// 检查缓存是否已经生成
var cachePath = dir + "/" + hash + ".cache"
var cachePathName = dir + "/" + hash
var cachePath = cachePathName + ".cache"
stat, err := os.Stat(cachePath)
if err == nil && time.Now().Sub(stat.ModTime()) <= 1*time.Second {
// 防止并发连续写入
@@ -388,16 +452,36 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
}
var tmpPath = cachePath + ".tmp"
if isPartial {
tmpPath = cachePath
tmpPath = cachePathName + ".cache"
}
// 先删除
err = this.list.Remove(hash)
if err != nil {
return nil, err
if !isPartial {
err = this.list.Remove(hash)
if err != nil {
return nil, err
}
}
writer, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_SYNC|os.O_WRONLY, 0666)
// 从已经存储的内容中读取信息
var isNewCreated = true
var partialBodyOffset int64
if isPartial {
readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
if err == nil {
var partialReader = NewPartialFileReader(readerFp)
err = partialReader.Init()
_ = partialReader.Close()
if err == nil && partialReader.bodyOffset > 0 {
isNewCreated = false
partialBodyOffset = partialReader.bodyOffset
} else {
_ = this.removeCacheFile(tmpPath)
}
}
}
writer, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return nil, err
}
@@ -424,22 +508,6 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
return nil, ErrFileIsWriting
}
// 是否已经有内容
var isNewCreated = true
var partialBodyOffset int64
if isPartial {
partialFP, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
if err == nil {
var partialReader = NewFileReader(partialFP)
err = partialReader.InitAutoDiscard(false)
if err == nil && partialReader.bodyOffset > 0 {
isNewCreated = false
partialBodyOffset = partialReader.bodyOffset
}
_ = partialReader.Close()
}
}
if isNewCreated {
err = writer.Truncate(0)
if err != nil {
@@ -502,13 +570,18 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
isOk = true
if isPartial {
return NewPartialFileWriter(writer, key, expiredAt, isNewCreated, isPartial, partialBodyOffset, func() {
ranges, err := NewPartialRangesFromFile(cachePathName + "@ranges.cache")
if err != nil {
ranges = NewPartialRanges()
}
return NewPartialFileWriter(writer, key, expiredAt, isNewCreated, isPartial, partialBodyOffset, ranges, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
sharedWritingFileKeyLocker.Unlock()
}), nil
} else {
return NewFileWriter(writer, key, expiredAt, func() {
return NewFileWriter(this, writer, key, expiredAt, -1, func() {
sharedWritingFileKeyLocker.Lock()
delete(sharedWritingFileKeyMap, key)
sharedWritingFileKeyLocker.Unlock()
@@ -518,9 +591,15 @@ func (this *FileStorage) OpenWriter(key string, expiredAt int64, status int, siz
// AddToList 添加到List
func (this *FileStorage) AddToList(item *Item) {
if this.memoryStorage != nil {
// 是否正在退出
if teaconst.IsQuiting {
return
}
var memoryStorage = this.memoryStorage
if memoryStorage != nil {
if item.Type == ItemTypeMemory {
this.memoryStorage.AddToList(item)
memoryStorage.AddToList(item)
return
}
}
@@ -535,23 +614,29 @@ func (this *FileStorage) AddToList(item *Item) {
// Delete 删除某个键值对应的缓存
func (this *FileStorage) Delete(key string) error {
// 是否正在退出
if teaconst.IsQuiting {
return nil
}
this.locker.Lock()
defer this.locker.Unlock()
// 先尝试内存缓存
if this.memoryStorage != nil {
_ = this.memoryStorage.Delete(key)
}
this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
_ = memoryStorage.Delete(key)
})
hash, path := this.keyPath(key)
err := this.list.Remove(hash)
if err != nil {
return err
}
err = os.Remove(path)
err = this.removeCacheFile(path)
if err == nil || os.IsNotExist(err) {
return nil
}
return err
}
@@ -571,9 +656,9 @@ func (this *FileStorage) CleanAll() error {
defer this.locker.Unlock()
// 先尝试内存缓存
if this.memoryStorage != nil {
_ = this.memoryStorage.CleanAll()
}
this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
_ = memoryStorage.CleanAll()
})
err := this.list.CleanAll()
if err != nil {
@@ -638,13 +723,18 @@ func (this *FileStorage) CleanAll() error {
// Purge 清理过期的缓存
func (this *FileStorage) Purge(keys []string, urlType string) error {
// 是否正在退出
if teaconst.IsQuiting {
return nil
}
this.locker.Lock()
defer this.locker.Unlock()
// 先尝试内存缓存
if this.memoryStorage != nil {
_ = this.memoryStorage.Purge(keys, urlType)
}
this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
_ = memoryStorage.Purge(keys, urlType)
})
// 目录
if urlType == "dir" {
@@ -659,10 +749,11 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
// 文件
for _, key := range keys {
hash, path := this.keyPath(key)
err := os.Remove(path)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
return err
}
err = this.list.Remove(hash)
if err != nil {
return err
@@ -679,9 +770,9 @@ func (this *FileStorage) Stop() {
defer this.locker.Unlock()
// 先尝试内存缓存
if this.memoryStorage != nil {
this.memoryStorage.Stop()
}
this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
memoryStorage.Stop()
})
_ = this.list.Reset()
if this.purgeTicker != nil {
@@ -696,6 +787,8 @@ func (this *FileStorage) Stop() {
if this.openFileCache != nil {
this.openFileCache.CloseAll()
}
this.ignoreKeys.Reset()
}
// TotalDiskSize 消耗的磁盘尺寸
@@ -705,21 +798,27 @@ func (this *FileStorage) TotalDiskSize() int64 {
// TotalMemorySize 内存尺寸
func (this *FileStorage) TotalMemorySize() int64 {
if this.memoryStorage == nil {
var memoryStorage = this.memoryStorage
if memoryStorage == nil {
return 0
}
return this.memoryStorage.TotalMemorySize()
return memoryStorage.TotalMemorySize()
}
// IgnoreKey 忽略某个Key即不缓存某个Key
func (this *FileStorage) IgnoreKey(key string) {
this.ignoreKeys.Push(key)
}
// 绝对路径
func (this *FileStorage) dir() string {
return this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/"
return this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/"
}
// 获取Key对应的文件路径
func (this *FileStorage) keyPath(key string) (hash string, path string) {
hash = stringutil.Md5(key)
dir := this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
dir := this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
path = dir + "/" + hash + ".cache"
return
}
@@ -729,7 +828,7 @@ func (this *FileStorage) hashPath(hash string) (path string) {
if len(hash) != 32 {
return ""
}
dir := this.cacheConfig.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
dir := this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
path = dir + "/" + hash + ".cache"
return
}
@@ -750,14 +849,22 @@ func (this *FileStorage) initList() error {
})**/
// 启动定时清理任务
var autoPurgeInterval = this.policy.PersistenceAutoPurgeInterval
if autoPurgeInterval <= 0 {
autoPurgeInterval = 30
if Tea.IsTesting() {
autoPurgeInterval = 10
}
this.initPurgeTicker()
// 热点处理任务
this.hotTicker = utils.NewTicker(1 * time.Minute)
if Tea.IsTesting() {
this.hotTicker = utils.NewTicker(10 * time.Second)
}
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
goman.New(func() {
for this.hotTicker.Next() {
trackers.Run("FILE_CACHE_STORAGE_HOT_LOOP", func() {
this.hotLoop()
})
}
})
// 退出时停止
events.OnKey(events.EventQuit, this, func() {
remotelogs.Println("CACHE", "quit clean timer")
@@ -774,26 +881,6 @@ func (this *FileStorage) initList() error {
}
}
})
goman.New(func() {
for this.purgeTicker.Next() {
trackers.Run("FILE_CACHE_STORAGE_PURGE_LOOP", func() {
this.purgeLoop()
})
}
})
// 热点处理任务
this.hotTicker = utils.NewTicker(1 * time.Minute)
if Tea.IsTesting() {
this.hotTicker = utils.NewTicker(10 * time.Second)
}
goman.New(func() {
for this.hotTicker.Next() {
trackers.Run("FILE_CACHE_STORAGE_HOT_LOOP", func() {
this.hotLoop()
})
}
})
return nil
}
@@ -837,10 +924,11 @@ func (this *FileStorage) purgeLoop() {
for i := 0; i < times; i++ {
countFound, err := this.list.Purge(purgeCount, func(hash string) error {
path := this.hashPath(hash)
err := os.Remove(path)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
return nil
})
if err != nil {
@@ -870,10 +958,11 @@ func (this *FileStorage) purgeLoop() {
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
err := this.list.PurgeLFU(count, func(hash string) error {
path := this.hashPath(hash)
err := os.Remove(path)
err := this.removeCacheFile(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
return nil
})
if err != nil {
@@ -923,18 +1012,33 @@ func (this *FileStorage) hotLoop() {
var buf = utils.BytePool16k.Get()
defer utils.BytePool16k.Put(buf)
for _, item := range result[:size] {
reader, err := this.openReader(item.Key, false, false)
reader, err := this.openReader(item.Key, false, false, false)
if err != nil {
continue
}
if reader == nil {
continue
}
if reader.ExpiresAt() <= time.Now().Unix() {
// 如果即将过期,则忽略
var nowUnixTime = time.Now().Unix()
if reader.ExpiresAt() <= nowUnixTime+600 {
continue
}
writer, err := this.memoryStorage.openWriter(item.Key, item.ExpiresAt, item.Status, reader.BodySize(), false)
// 计算合适的过期时间
var bestExpiresAt = nowUnixTime + HotItemLifeSeconds
var hotTimes = int64(item.Hits) / 1000
if hotTimes > 8 {
hotTimes = 8
}
bestExpiresAt += hotTimes * HotItemLifeSeconds
var expiresAt = reader.ExpiresAt()
if expiresAt <= 0 || expiresAt > bestExpiresAt {
expiresAt = bestExpiresAt
}
writer, err := memoryStorage.openWriter(item.Key, expiresAt, reader.Status(), reader.BodySize(), -1, false)
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "transfer hot item failed: "+err.Error())
@@ -970,10 +1074,10 @@ func (this *FileStorage) hotLoop() {
continue
}
this.memoryStorage.AddToList(&Item{
memoryStorage.AddToList(&Item{
Type: writer.ItemType(),
Key: item.Key,
ExpiredAt: item.ExpiresAt,
ExpiredAt: expiresAt,
HeaderSize: writer.HeaderSize(),
BodySize: writer.BodySize(),
})
@@ -985,9 +1089,9 @@ func (this *FileStorage) hotLoop() {
}
func (this *FileStorage) diskCapacityBytes() int64 {
c1 := this.policy.CapacityBytes()
var c1 = this.policy.CapacityBytes()
if SharedManager.MaxDiskCapacity != nil {
c2 := SharedManager.MaxDiskCapacity.Bytes()
var c2 = SharedManager.MaxDiskCapacity.Bytes()
if c2 > 0 {
return c2
}
@@ -1025,3 +1129,141 @@ func (this *FileStorage) cleanDeletedDirs(dir string) error {
}
return nil
}
// 增加某个Key的点击量
func (this *FileStorage) increaseHit(key string, hash string, reader Reader) {
var rate = this.policy.PersistenceHitSampleRate
if rate <= 0 {
rate = 1000
}
if this.lastHotSize == 0 {
// 自动降低采样率来增加热点数据的缓存几率
rate = rate / 10
}
if rands.Int(0, rate) == 0 {
var memoryStorage = this.memoryStorage
var hitErr = this.list.IncreaseHit(hash)
if hitErr != nil {
// 此错误可以忽略
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
}
// 增加到热点
// 这里不收录缓存尺寸过大的文件
if memoryStorage != nil && reader.BodySize() > 0 && reader.BodySize() < 128*1024*1024 {
this.hotMapLocker.Lock()
hotItem, ok := this.hotMap[key]
// 限制热点数据存活时间
var unixTime = time.Now().Unix()
var expiresAt = reader.ExpiresAt()
if expiresAt <= 0 || expiresAt > unixTime+HotItemLifeSeconds {
expiresAt = unixTime + HotItemLifeSeconds
}
if ok {
hotItem.Hits++
} else if len(this.hotMap) < HotItemSize { // 控制数量
this.hotMap[key] = &HotItem{
Key: key,
Hits: 1,
}
}
this.hotMapLocker.Unlock()
}
}
}
// 删除缓存文件
func (this *FileStorage) removeCacheFile(path string) error {
var err = os.Remove(path)
if err == nil || os.IsNotExist(err) {
err = nil
// 删除Partial相关
_ = os.Remove(partialRangesFilePath(path))
}
return err
}
// 创建当前策略包含的内存缓存
func (this *FileStorage) createMemoryStorage() error {
var memoryPolicy = &serverconfigs.HTTPCachePolicy{
Id: this.policy.Id,
IsOn: this.policy.IsOn,
Name: this.policy.Name,
Description: this.policy.Description,
Capacity: this.options.MemoryPolicy.Capacity,
MaxKeys: this.policy.MaxKeys,
MaxSize: &shared.SizeCapacity{Count: 128, Unit: shared.SizeCapacityUnitMB}, // TODO 将来可以修改
Type: serverconfigs.CachePolicyStorageMemory,
Options: this.policy.Options,
Life: this.policy.Life,
MinLife: this.policy.MinLife,
MaxLife: this.policy.MaxLife,
MemoryAutoPurgeCount: this.policy.MemoryAutoPurgeCount,
MemoryAutoPurgeInterval: this.policy.MemoryAutoPurgeInterval,
MemoryLFUFreePercent: this.policy.MemoryLFUFreePercent,
}
err := memoryPolicy.Init()
if err != nil {
return err
}
var memoryStorage = NewMemoryStorage(memoryPolicy, this)
err = memoryStorage.Init()
if err != nil {
return err
}
this.memoryStorage = memoryStorage
return nil
}
func (this *FileStorage) initPurgeTicker() {
var autoPurgeInterval = this.policy.PersistenceAutoPurgeInterval
if autoPurgeInterval <= 0 {
autoPurgeInterval = 30
if Tea.IsTesting() {
autoPurgeInterval = 10
}
}
if this.purgeTicker != nil {
this.purgeTicker.Stop()
}
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
goman.New(func() {
for this.purgeTicker.Next() {
trackers.Run("FILE_CACHE_STORAGE_PURGE_LOOP", func() {
this.purgeLoop()
})
}
})
}
func (this *FileStorage) initOpenFileCache() {
var err error
var oldOpenFileCache = this.openFileCache
// 启用新的
if this.options.OpenFileCache != nil && this.options.OpenFileCache.IsOn && this.options.OpenFileCache.Max > 0 {
this.openFileCache, err = NewOpenFileCache(this.options.OpenFileCache.Max)
if err != nil {
remotelogs.Error("CACHE", "open file cache failed: "+err.Error())
}
}
// 关闭老的
if oldOpenFileCache != nil {
oldOpenFileCache.CloseAll()
}
}
func (this *FileStorage) runMemoryStorageSafety(f func(memoryStorage *MemoryStorage)) {
var memoryStorage = this.memoryStorage
if memoryStorage != nil {
f(memoryStorage)
}
}

View File

@@ -62,7 +62,7 @@ func TestFileStorage_OpenWriter(t *testing.T) {
header := []byte("Header")
body := []byte("This is Body")
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, false)
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -100,7 +100,7 @@ func TestFileStorage_OpenWriter_Partial(t *testing.T) {
t.Fatal(err)
}
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, true)
writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, true)
if err != nil {
t.Fatal(err)
}
@@ -110,7 +110,7 @@ func TestFileStorage_OpenWriter_Partial(t *testing.T) {
t.Fatal(err)
}
err = writer.WriteAt([]byte("Hello, World"), 0)
err = writer.WriteAt(0, []byte("Hello, World"))
if err != nil {
t.Fatal(err)
}
@@ -139,7 +139,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
t.Log(time.Since(now).Seconds()*1000, "ms")
}()
writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, false)
writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -212,7 +212,7 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
go func(i int) {
defer wg.Done()
writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, false)
writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, false)
if err != nil {
if err != ErrFileIsWriting {
t.Error(err)
@@ -267,7 +267,7 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
go func(i int) {
defer wg.Done()
writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, false)
writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, false)
if err != nil {
if err != ErrFileIsWriting {
t.Error(err)
@@ -311,7 +311,7 @@ func TestFileStorage_Read(t *testing.T) {
t.Fatal(err)
}
now := time.Now()
reader, err := storage.OpenReader("my-key", false)
reader, err := storage.OpenReader("my-key", false, false)
if err != nil {
t.Fatal(err)
}
@@ -347,7 +347,7 @@ func TestFileStorage_Read_HTTP_Response(t *testing.T) {
t.Fatal(err)
}
now := time.Now()
reader, err := storage.OpenReader("my-http-response", false)
reader, err := storage.OpenReader("my-http-response", false, false)
if err != nil {
t.Fatal(err)
}
@@ -401,7 +401,7 @@ func TestFileStorage_Read_NotFound(t *testing.T) {
}
now := time.Now()
buf := make([]byte, 6)
reader, err := storage.OpenReader("my-key-10000", false)
reader, err := storage.OpenReader("my-key-10000", false, false)
if err != nil {
if err == ErrNotFound {
t.Log("cache not fund")
@@ -511,7 +511,7 @@ func TestFileStorage_Stop(t *testing.T) {
}
func TestFileStorage_DecodeFile(t *testing.T) {
storage := NewFileStorage(&serverconfigs.HTTPCachePolicy{
var storage = NewFileStorage(&serverconfigs.HTTPCachePolicy{
Id: 1,
IsOn: true,
Options: map[string]interface{}{
@@ -526,6 +526,11 @@ func TestFileStorage_DecodeFile(t *testing.T) {
t.Log(path)
}
func TestFileStorage_RemoveCacheFile(t *testing.T) {
var storage = NewFileStorage(nil)
t.Log(storage.removeCacheFile("/Users/WorkSpace/EdgeProject/EdgeCache/p43/15/7e/157eba0dfc6dfb6fbbf20b1f9e584674.cache"))
}
func BenchmarkFileStorage_Read(b *testing.B) {
runtime.GOMAXPROCS(1)
@@ -543,7 +548,7 @@ func BenchmarkFileStorage_Read(b *testing.B) {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
reader, err := storage.OpenReader("my-key", false)
reader, err := storage.OpenReader("my-key", false, false)
if err != nil {
b.Fatal(err)
}
@@ -559,8 +564,8 @@ func BenchmarkFileStorage_KeyPath(b *testing.B) {
runtime.GOMAXPROCS(1)
var storage = &FileStorage{
cacheConfig: &serverconfigs.HTTPFileCacheStorage{},
policy: &serverconfigs.HTTPCachePolicy{Id: 1},
options: &serverconfigs.HTTPFileCacheStorage{},
policy: &serverconfigs.HTTPCachePolicy{Id: 1},
}
for i := 0; i < b.N; i++ {

View File

@@ -10,10 +10,11 @@ type StorageInterface interface {
Init() error
// OpenReader 读取缓存
OpenReader(key string, useStale bool) (reader Reader, err error)
OpenReader(key string, useStale bool, isPartial bool) (reader Reader, err error)
// OpenWriter 打开缓存写入器等待写入
OpenWriter(key string, expiredAt int64, status int, size int64, isPartial bool) (Writer, error)
// size 和 maxSize 可能为-1
OpenWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error)
// Delete 删除某个键值对应的缓存
Delete(key string) error
@@ -39,6 +40,15 @@ type StorageInterface interface {
// Policy 获取当前存储的Policy
Policy() *serverconfigs.HTTPCachePolicy
// UpdatePolicy 修改策略
UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy)
// CanUpdatePolicy 检查策略是否可以更新
CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool
// AddToList 将缓存添加到列表
AddToList(item *Item)
// IgnoreKey 忽略某个Key即不缓存某个Key
IgnoreKey(key string)
}

View File

@@ -7,10 +7,13 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
"github.com/TeaOSLab/EdgeNode/internal/utils/sizes"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"github.com/shirou/gopsutil/v3/load"
"math"
"runtime"
"strconv"
@@ -40,31 +43,39 @@ type MemoryStorage struct {
locker *sync.RWMutex
valuesMap map[uint64]*MemoryItem // hash => item
dirtyChan chan string // hash chan
dirtyChan chan string // hash chan
dirtyQueueSize int
purgeTicker *utils.Ticker
totalSize int64
writingKeyMap map[string]zero.Zero // key => bool
ignoreKeys *setutils.FixedSet
}
func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage StorageInterface) *MemoryStorage {
var dirtyChan chan string
var queueSize = policy.MemoryAutoFlushQueueSize
if parentStorage != nil {
var queueSize = policy.MemoryAutoFlushQueueSize
if queueSize <= 0 {
queueSize = 2048
queueSize = 2048 + int(policy.CapacityBytes()/sizes.G)*2048
}
dirtyChan = make(chan string, queueSize)
}
return &MemoryStorage{
parentStorage: parentStorage,
policy: policy,
list: NewMemoryList(),
locker: &sync.RWMutex{},
valuesMap: map[uint64]*MemoryItem{},
dirtyChan: dirtyChan,
writingKeyMap: map[string]zero.Zero{},
parentStorage: parentStorage,
policy: policy,
list: NewMemoryList(),
locker: &sync.RWMutex{},
valuesMap: map[uint64]*MemoryItem{},
dirtyChan: dirtyChan,
dirtyQueueSize: queueSize,
writingKeyMap: map[string]zero.Zero{},
ignoreKeys: setutils.NewFixedSet(32768),
}
}
@@ -79,33 +90,25 @@ func (this *MemoryStorage) Init() error {
atomic.AddInt64(&this.totalSize, -item.TotalSize())
})
var autoPurgeInterval = this.policy.MemoryAutoPurgeInterval
if autoPurgeInterval <= 0 {
autoPurgeInterval = 5
}
// 启动定时清理任务
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
goman.New(func() {
for this.purgeTicker.Next() {
var tr = trackers.Begin("MEMORY_CACHE_STORAGE_PURGE_LOOP")
this.purgeLoop()
tr.End()
}
})
this.initPurgeTicker()
// 启动定时Flush memory to disk任务
goman.New(func() {
for hash := range this.dirtyChan {
this.flushItem(hash)
if this.parentStorage != nil {
// TODO 应该根据磁盘性能决定线程数
var threads = 1
for i := 0; i < threads; i++ {
goman.New(func() {
this.startFlush()
})
}
})
}
return nil
}
// OpenReader 读取缓存
func (this *MemoryStorage) OpenReader(key string, useStale bool) (Reader, error) {
func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool) (Reader, error) {
hash := this.hash(key)
this.locker.RLock()
@@ -145,15 +148,27 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool) (Reader, error)
}
// OpenWriter 打开缓存写入器等待写入
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, size int64, isPartial bool) (Writer, error) {
func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error) {
if this.ignoreKeys.Has(key) {
return nil, ErrEntityTooLarge
}
// TODO 内存缓存暂时不支持分块内容存储
if isPartial {
return nil, ErrFileIsWriting
}
return this.openWriter(key, expiredAt, status, size, true)
return this.openWriter(key, expiredAt, status, size, maxSize, true)
}
func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, size int64, isDirty bool) (Writer, error) {
func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isDirty bool) (Writer, error) {
// 待写入队列是否已满
if isDirty &&
this.parentStorage != nil &&
this.dirtyQueueSize > 0 &&
len(this.dirtyChan) == this.dirtyQueueSize { // 缓存时间过长
return nil, ErrWritingQueueFull
}
this.locker.Lock()
defer this.locker.Unlock()
@@ -200,7 +215,7 @@ func (this *MemoryStorage) openWriter(key string, expiredAt int64, status int, s
}
isWriting = true
return NewMemoryWriter(this, key, expiredAt, status, isDirty, func() {
return NewMemoryWriter(this, key, expiredAt, status, isDirty, maxSize, func() {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
@@ -269,7 +284,7 @@ func (this *MemoryStorage) Stop() {
this.purgeTicker.Stop()
}
if this.parentStorage != nil && this.dirtyChan != nil {
if this.dirtyChan != nil {
close(this.dirtyChan)
}
@@ -277,6 +292,8 @@ func (this *MemoryStorage) Stop() {
this.locker.Unlock()
this.ignoreKeys.Reset()
// 回收内存
runtime.GC()
@@ -288,6 +305,26 @@ func (this *MemoryStorage) Policy() *serverconfigs.HTTPCachePolicy {
return this.policy
}
// UpdatePolicy 修改策略
func (this *MemoryStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) {
var oldPolicy = this.policy
this.policy = newPolicy
if oldPolicy.MemoryAutoPurgeInterval != newPolicy.MemoryAutoPurgeInterval {
this.initPurgeTicker()
}
// 如果是空的,则清空
if newPolicy.CapacityBytes() == 0 {
_ = this.CleanAll()
}
}
// CanUpdatePolicy 检查策略是否可以更新
func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy) bool {
return true
}
// AddToList 将缓存添加到列表
func (this *MemoryStorage) AddToList(item *Item) {
item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/
@@ -305,6 +342,11 @@ func (this *MemoryStorage) TotalMemorySize() int64 {
return atomic.LoadInt64(&this.totalSize)
}
// IgnoreKey 忽略某个Key即不缓存某个Key
func (this *MemoryStorage) IgnoreKey(key string) {
this.ignoreKeys.Push(key)
}
// 计算Key Hash
func (this *MemoryStorage) hash(key string) uint64 {
return xxhash.Sum64String(key)
@@ -373,7 +415,40 @@ func (this *MemoryStorage) purgeLoop() {
}
}
// Flush任务
// 开始Flush任务
func (this *MemoryStorage) startFlush() {
var statCount = 0
var writeDelayMS float64 = 0
for hash := range this.dirtyChan {
statCount++
if statCount == 100 {
statCount = 0
loadStat, err := load.Avg()
if err == nil && loadStat != nil {
if loadStat.Load1 > 10 {
writeDelayMS = 100
} else if loadStat.Load1 > 3 {
writeDelayMS = 50
} else if loadStat.Load1 > 2 {
writeDelayMS = 10
} else {
writeDelayMS = 0
}
}
}
this.flushItem(hash)
if writeDelayMS > 0 {
time.Sleep(time.Duration(writeDelayMS) * time.Millisecond)
}
}
}
// 单次Flush任务
func (this *MemoryStorage) flushItem(key string) {
if this.parentStorage == nil {
return
@@ -391,7 +466,7 @@ func (this *MemoryStorage) flushItem(key string) {
return
}
writer, err := this.parentStorage.OpenWriter(key, item.ExpiredAt, item.Status, -1, false)
writer, err := this.parentStorage.OpenWriter(key, item.ExpiredAt, item.Status, -1, -1, false)
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "flush items failed: open writer failed: "+err.Error())
@@ -417,6 +492,7 @@ func (this *MemoryStorage) flushItem(key string) {
if err != nil {
_ = writer.Discard()
remotelogs.Error("CACHE", "flush items failed: close writer failed: "+err.Error())
return
}
this.parentStorage.AddToList(&Item{
@@ -427,6 +503,9 @@ func (this *MemoryStorage) flushItem(key string) {
BodySize: writer.BodySize(),
})
// 从内存中移除
_ = this.Delete(key)
return
}
@@ -453,3 +532,25 @@ func (this *MemoryStorage) deleteWithoutLocker(key string) error {
_ = this.list.Remove(fmt.Sprintf("%d", hash))
return nil
}
func (this *MemoryStorage) initPurgeTicker() {
var autoPurgeInterval = this.policy.MemoryAutoPurgeInterval
if autoPurgeInterval <= 0 {
autoPurgeInterval = 5
}
// 启动定时清理任务
if this.purgeTicker != nil {
this.purgeTicker.Stop()
}
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
goman.New(func() {
for this.purgeTicker.Next() {
var tr = trackers.Begin("MEMORY_CACHE_STORAGE_PURGE_LOOP")
this.purgeLoop()
tr.End()
}
})
}

View File

@@ -15,7 +15,7 @@ import (
func TestMemoryStorage_OpenWriter(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, false)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -25,7 +25,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
t.Log(storage.valuesMap)
{
reader, err := storage.OpenReader("abc", false)
reader, err := storage.OpenReader("abc", false, false)
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc")
@@ -52,7 +52,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
}
{
_, err := storage.OpenReader("abc 2", false)
_, err := storage.OpenReader("abc 2", false, false)
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc2")
@@ -62,13 +62,13 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
}
}
writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, false)
writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
_, _ = writer.Write([]byte("Hello123"))
{
reader, err := storage.OpenReader("abc", false)
reader, err := storage.OpenReader("abc", false, false)
if err != nil {
if err == ErrNotFound {
t.Log("not found: abc")
@@ -97,13 +97,13 @@ func TestMemoryStorage_OpenReaderLock(t *testing.T) {
IsDone: true,
},
}
_, _ = storage.OpenReader("test", false)
_, _ = storage.OpenReader("test", false, false)
}
func TestMemoryStorage_Delete(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
{
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, false)
writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -111,7 +111,7 @@ func TestMemoryStorage_Delete(t *testing.T) {
t.Log(len(storage.valuesMap))
}
{
writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, false)
writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -126,7 +126,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, false)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -139,7 +139,7 @@ func TestMemoryStorage_Stat(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, false)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -163,7 +163,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, false)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -175,7 +175,7 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, false)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -198,7 +198,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
expiredAt := time.Now().Unix() + 60
{
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, false)
writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -210,7 +210,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
})
}
{
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, false)
writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}
@@ -241,7 +241,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
for i := 0; i < 1000; i++ {
expiredAt := time.Now().Unix() + int64(rands.Int(0, 60))
key := "abc" + strconv.Itoa(i)
writer, err := storage.OpenWriter(key, expiredAt, 200, -1, false)
writer, err := storage.OpenWriter(key, expiredAt, 200, -1, -1, false)
if err != nil {
t.Fatal(err)
}

View File

@@ -0,0 +1,18 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import "strings"
// 获取 ranges 文件路径
func partialRangesFilePath(path string) string {
// ranges路径
var dotIndex = strings.LastIndex(path, ".")
var rangePath = ""
if dotIndex < 0 {
rangePath = path + "@ranges.cache"
} else {
rangePath = path[:dotIndex] + "@ranges" + path[dotIndex:]
}
return rangePath
}

View File

@@ -9,7 +9,7 @@ type Writer interface {
Write(data []byte) (n int, err error)
// WriteAt 在指定位置写入数据
WriteAt(data []byte, offset int64) error
WriteAt(offset int64, data []byte) error
// HeaderSize 写入的Header数据大小
HeaderSize() int64

View File

@@ -11,20 +11,24 @@ import (
)
type FileWriter struct {
storage StorageInterface
rawWriter *os.File
key string
headerSize int64
bodySize int64
expiredAt int64
maxSize int64
endFunc func()
once sync.Once
}
func NewFileWriter(rawWriter *os.File, key string, expiredAt int64, endFunc func()) *FileWriter {
func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, expiredAt int64, maxSize int64, endFunc func()) *FileWriter {
return &FileWriter{
storage: storage,
key: key,
rawWriter: rawWriter,
expiredAt: expiredAt,
maxSize: maxSize,
endFunc: endFunc,
}
}
@@ -60,6 +64,15 @@ func (this *FileWriter) WriteHeaderLength(headerLength int) error {
func (this *FileWriter) Write(data []byte) (n int, err error) {
n, err = this.rawWriter.Write(data)
this.bodySize += int64(n)
if this.maxSize > 0 && this.bodySize > this.maxSize {
err = ErrEntityTooLarge
if this.storage != nil {
this.storage.IgnoreKey(this.key)
}
}
if err != nil {
_ = this.Discard()
}
@@ -67,7 +80,7 @@ func (this *FileWriter) Write(data []byte) (n int, err error) {
}
// WriteAt 在指定位置写入数据
func (this *FileWriter) WriteAt(data []byte, offset int64) error {
func (this *FileWriter) WriteAt(offset int64, data []byte) error {
_ = data
_ = offset
return errors.New("not supported")

View File

@@ -16,6 +16,7 @@ type MemoryWriter struct {
bodySize int64
status int
isDirty bool
maxSize int64
hash uint64
item *MemoryItem
@@ -23,7 +24,7 @@ type MemoryWriter struct {
once sync.Once
}
func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, endFunc func()) *MemoryWriter {
func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, isDirty bool, maxSize int64, endFunc func()) *MemoryWriter {
w := &MemoryWriter{
storage: memoryStorage,
key: key,
@@ -35,6 +36,7 @@ func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64,
},
status: status,
isDirty: isDirty,
maxSize: maxSize,
endFunc: endFunc,
}
w.hash = w.calculateHash(key)
@@ -53,11 +55,19 @@ func (this *MemoryWriter) WriteHeader(data []byte) (n int, err error) {
func (this *MemoryWriter) Write(data []byte) (n int, err error) {
this.bodySize += int64(len(data))
this.item.BodyValue = append(this.item.BodyValue, data...)
// 检查尺寸
if this.maxSize > 0 && this.bodySize > this.maxSize {
err = ErrEntityTooLarge
this.storage.IgnoreKey(this.key)
return len(data), err
}
return len(data), nil
}
// WriteAt 在指定位置写入数据
func (this *MemoryWriter) WriteAt(b []byte, offset int64) error {
func (this *MemoryWriter) WriteAt(offset int64, b []byte) error {
_ = b
_ = offset
return errors.New("not supported")

View File

@@ -7,7 +7,6 @@ import (
"github.com/iwind/TeaGo/types"
"io"
"os"
"strings"
"sync"
)
@@ -23,9 +22,12 @@ type PartialFileWriter struct {
isNew bool
isPartial bool
bodyOffset int64
ranges *PartialRanges
rangePath string
}
func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, isNew bool, isPartial bool, bodyOffset int64, endFunc func()) *PartialFileWriter {
func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
return &PartialFileWriter{
key: key,
rawWriter: rawWriter,
@@ -34,6 +36,8 @@ func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, isNew
isNew: isNew,
isPartial: isPartial,
bodyOffset: bodyOffset,
ranges: ranges,
rangePath: partialRangesFilePath(rawWriter.Name()),
}
}
@@ -50,6 +54,21 @@ func (this *PartialFileWriter) WriteHeader(data []byte) (n int, err error) {
return
}
func (this *PartialFileWriter) AppendHeader(data []byte) error {
_, err := this.rawWriter.Write(data)
if err != nil {
_ = this.Discard()
} else {
var c = len(data)
this.headerSize += int64(c)
err = this.WriteHeaderLength(int(this.headerSize))
if err != nil {
_ = this.Discard()
}
}
return err
}
// WriteHeaderLength 写入Header长度数据
func (this *PartialFileWriter) WriteHeaderLength(headerLength int) error {
bytes4 := make([]byte, 4)
@@ -78,12 +97,34 @@ func (this *PartialFileWriter) Write(data []byte) (n int, err error) {
}
// WriteAt 在指定位置写入数据
func (this *PartialFileWriter) WriteAt(data []byte, offset int64) error {
func (this *PartialFileWriter) WriteAt(offset int64, data []byte) error {
var c = int64(len(data))
if c == 0 {
return nil
}
var end = offset + c - 1
// 是否已包含在内
if this.ranges.Contains(offset, end) {
return nil
}
if this.bodyOffset == 0 {
this.bodyOffset = SizeMeta + int64(len(this.key)) + this.headerSize
}
_, err := this.rawWriter.WriteAt(data, this.bodyOffset+offset)
return err
if err != nil {
return err
}
this.ranges.Add(offset, end)
return nil
}
// SetBodyLength 设置内容总长度
func (this *PartialFileWriter) SetBodyLength(bodyLength int64) {
this.bodySize = bodyLength
}
// WriteBodyLength 写入Body长度数据
@@ -109,31 +150,30 @@ func (this *PartialFileWriter) Close() error {
this.endFunc()
})
var path = this.rawWriter.Name()
err := this.ranges.WriteToFile(this.rangePath)
if err != nil {
return err
}
// 关闭当前writer
if this.isNew {
err := this.WriteHeaderLength(types.Int(this.headerSize))
err = this.WriteHeaderLength(types.Int(this.headerSize))
if err != nil {
_ = this.rawWriter.Close()
_ = os.Remove(path)
this.remove()
return err
}
err = this.WriteBodyLength(this.bodySize)
if err != nil {
_ = this.rawWriter.Close()
_ = os.Remove(path)
this.remove()
return err
}
}
err := this.rawWriter.Close()
err = this.rawWriter.Close()
if err != nil {
_ = os.Remove(path)
} else if !this.isPartial {
err = os.Rename(path, strings.Replace(path, ".tmp", "", 1))
if err != nil {
_ = os.Remove(path)
}
this.remove()
}
return err
@@ -147,6 +187,8 @@ func (this *PartialFileWriter) Discard() error {
_ = this.rawWriter.Close()
_ = os.Remove(this.rangePath)
err := os.Remove(this.rawWriter.Name())
return err
}
@@ -171,3 +213,12 @@ func (this *PartialFileWriter) Key() string {
func (this *PartialFileWriter) ItemType() ItemType {
return ItemTypeFile
}
func (this *PartialFileWriter) IsNew() bool {
return this.isNew && len(this.ranges.Ranges) == 0
}
func (this *PartialFileWriter) remove() {
_ = os.Remove(this.rawWriter.Name())
_ = os.Remove(this.rangePath)
}

View File

@@ -11,8 +11,8 @@ import (
"time"
)
func TestPartialFileWriter_SeekOffset(t *testing.T) {
var path = "/tmp/test@partial.cache"
func TestPartialFileWriter_Write(t *testing.T) {
var path = "/tmp/test_partial.cache"
_ = os.Remove(path)
var reader = func() {
@@ -27,7 +27,8 @@ func TestPartialFileWriter_SeekOffset(t *testing.T) {
if err != nil {
t.Fatal(err)
}
var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, true, true, 0, func() {
var ranges = caches.NewPartialRanges()
var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, true, true, 0, ranges, func() {
t.Log("end")
})
_, err = writer.WriteHeader([]byte("header"))
@@ -36,7 +37,7 @@ func TestPartialFileWriter_SeekOffset(t *testing.T) {
}
// 移动位置
err = writer.WriteAt([]byte("HELLO"), 100)
err = writer.WriteAt(100, []byte("HELLO"))
if err != nil {
t.Fatal(err)
}

View File

@@ -2,7 +2,14 @@
package compressions
import "io"
type Reader interface {
Read(p []byte) (n int, err error)
Reset(reader io.Reader) error
RawClose() error
Close() error
SetPool(pool *ReaderPool)
ResetFinish()
}

View File

@@ -0,0 +1,26 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
type BaseReader struct {
pool *ReaderPool
isFinished bool
}
func (this *BaseReader) SetPool(pool *ReaderPool) {
this.pool = pool
}
func (this *BaseReader) Finish(obj Reader) error {
err := obj.RawClose()
if err == nil && this.pool != nil && !this.isFinished {
this.pool.Put(obj)
}
this.isFinished = true
return err
}
func (this *BaseReader) ResetFinish() {
this.isFinished = false
}

View File

@@ -9,10 +9,16 @@ import (
)
type BrotliReader struct {
BaseReader
reader *brotli.Reader
}
func NewBrotliReader(reader io.Reader) (Reader, error) {
return sharedBrotliReaderPool.Get(reader)
}
func newBrotliReader(reader io.Reader) (Reader, error) {
return &BrotliReader{reader: brotli.NewReader(reader)}, nil
}
@@ -24,6 +30,14 @@ func (this *BrotliReader) Read(p []byte) (n int, err error) {
return
}
func (this *BrotliReader) Close() error {
func (this *BrotliReader) Reset(reader io.Reader) error {
return this.reader.Reset(reader)
}
func (this *BrotliReader) RawClose() error {
return nil
}
func (this *BrotliReader) Close() error {
return this.Finish(this)
}

View File

@@ -0,0 +1,51 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"io"
"testing"
)
func TestBrotliReader(t *testing.T) {
for _, testString := range []string{"Hello", "World", "Ni", "Hao"} {
t.Log("===", testString, "===")
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
t.Fatal(err)
}
_, err = writer.Write([]byte(testString))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := compressions.NewBrotliReader(buf)
if err != nil {
t.Fatal(err)
}
var data = make([]byte, 4096)
for {
n, err := reader.Read(data)
if n > 0 {
t.Log(string(data[:n]))
}
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
}

View File

@@ -8,10 +8,16 @@ import (
)
type DeflateReader struct {
BaseReader
reader io.ReadCloser
}
func NewDeflateReader(reader io.Reader) (Reader, error) {
return sharedDeflateReaderPool.Get(reader)
}
func newDeflateReader(reader io.Reader) (Reader, error) {
return &DeflateReader{reader: flate.NewReader(reader)}, nil
}
@@ -19,6 +25,15 @@ func (this *DeflateReader) Read(p []byte) (n int, err error) {
return this.reader.Read(p)
}
func (this *DeflateReader) Close() error {
func (this *DeflateReader) Reset(reader io.Reader) error {
this.reader = flate.NewReader(reader)
return nil
}
func (this *DeflateReader) RawClose() error {
return this.reader.Close()
}
func (this *DeflateReader) Close() error {
return this.Finish(this)
}

View File

@@ -0,0 +1,51 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"io"
"testing"
)
func TestDeflateReader(t *testing.T) {
for _, testString := range []string{"Hello", "World", "Ni", "Hao"} {
t.Log("===", testString, "===")
var buf = &bytes.Buffer{}
writer, err := compressions.NewDeflateWriter(buf, 5)
if err != nil {
t.Fatal(err)
}
_, err = writer.Write([]byte(testString))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := compressions.NewDeflateReader(buf)
if err != nil {
t.Fatal(err)
}
var data = make([]byte, 4096)
for {
n, err := reader.Read(data)
if n > 0 {
t.Log(string(data[:n]))
}
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
}

View File

@@ -8,10 +8,16 @@ import (
)
type GzipReader struct {
BaseReader
reader *gzip.Reader
}
func NewGzipReader(reader io.Reader) (Reader, error) {
return sharedGzipReaderPool.Get(reader)
}
func newGzipReader(reader io.Reader) (Reader, error) {
r, err := gzip.NewReader(reader)
if err != nil {
return nil, err
@@ -25,6 +31,14 @@ func (this *GzipReader) Read(p []byte) (n int, err error) {
return this.reader.Read(p)
}
func (this *GzipReader) Close() error {
func (this *GzipReader) Reset(reader io.Reader) error {
return this.reader.Reset(reader)
}
func (this *GzipReader) RawClose() error {
return this.reader.Close()
}
func (this *GzipReader) Close() error {
return this.Finish(this)
}

View File

@@ -1,68 +1,106 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
package compressions_test
import (
"bytes"
"errors"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"io"
"os"
"strings"
"testing"
)
func TestGzipReader(t *testing.T) {
fp, err := os.Open("/Users/WorkSpace/EdgeProject/EdgeCache/p43/36/7e/367e02720713fe05b66573a1d69b4f0a.cache")
if err != nil {
// not fatal
t.Log(err)
return
}
defer func() {
_ = fp.Close()
}()
var buf = make([]byte, 32*1024)
cacheReader := caches.NewFileReader(fp)
err = cacheReader.Init()
if err != nil {
t.Fatal(err)
}
var headerBuf = []byte{}
err = cacheReader.ReadHeader(buf, func(n int) (goNext bool, err error) {
headerBuf = append(headerBuf, buf[:n]...)
for {
nIndex := bytes.Index(headerBuf, []byte{'\n'})
if nIndex >= 0 {
row := headerBuf[:nIndex]
spaceIndex := bytes.Index(row, []byte{':'})
if spaceIndex <= 0 {
return false, errors.New("invalid header '" + string(row) + "'")
}
headerBuf = headerBuf[nIndex+1:]
} else {
break
}
}
return true, nil
})
reader, err := NewGzipReader(cacheReader)
if err != nil {
t.Fatal(err)
}
for {
n, err := reader.Read(buf)
for _, testString := range []string{"Hello", "World", "Ni", "Hao"} {
t.Log("===", testString, "===")
var buf = &bytes.Buffer{}
writer, err := compressions.NewGzipWriter(buf, 5)
if err != nil {
if err != io.EOF {
t.Fatal(err)
}
_, err = writer.Write([]byte(testString))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := compressions.NewGzipReader(buf)
if err != nil {
t.Fatal(err)
}
var data = make([]byte, 4096)
for {
n, err := reader.Read(data)
if n > 0 {
t.Log(string(data[:n]))
}
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
} else {
break
}
}
t.Log(string(buf[:n]))
_ = n
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
}
func BenchmarkGzipReader(b *testing.B) {
var randomData = func() []byte {
var b = strings.Builder{}
for i := 0; i < 1024; i++ {
b.WriteString(types.String(rands.Int64() % 10))
}
return []byte(b.String())
}
var buf = &bytes.Buffer{}
writer, err := compressions.NewGzipWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
_, err = writer.Write(randomData())
if err != nil {
b.Fatal(err)
}
err = writer.Close()
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
var newBytes = make([]byte, buf.Len())
copy(newBytes, buf.Bytes())
reader, err := compressions.NewGzipReader(bytes.NewReader(newBytes))
if err != nil {
b.Fatal(err)
}
var data = make([]byte, 4096)
for {
n, err := reader.Read(data)
if n > 0 {
_ = data[:n]
}
if err != nil {
if err == io.EOF {
break
}
b.Fatal(err)
}
}
err = reader.Close()
if err != nil {
b.Fatal(err)
}
}
}

View File

@@ -0,0 +1,56 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"io"
)
type ReaderPool struct {
c chan Reader
newFunc func(reader io.Reader) (Reader, error)
}
func NewReaderPool(maxSize int, newFunc func(reader io.Reader) (Reader, error)) *ReaderPool {
if maxSize <= 0 {
maxSize = 1024
}
return &ReaderPool{
c: make(chan Reader, maxSize),
newFunc: newFunc,
}
}
func (this *ReaderPool) Get(parentReader io.Reader) (Reader, error) {
select {
case reader := <-this.c:
err := reader.Reset(parentReader)
if err != nil {
// create new
reader, err = this.newFunc(parentReader)
if err != nil {
return nil, err
}
reader.SetPool(this)
return reader, nil
}
reader.ResetFinish()
return reader, nil
default:
// create new
reader, err := this.newFunc(parentReader)
if err != nil {
return nil, err
}
reader.SetPool(this)
return reader, nil
}
}
func (this *ReaderPool) Put(reader Reader) {
select {
case this.c <- reader:
default:
}
}

View File

@@ -0,0 +1,20 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
var sharedBrotliReaderPool *ReaderPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedBrotliReaderPool = NewReaderPool(maxSize, func(reader io.Reader) (Reader, error) {
return newBrotliReader(reader)
})
}

View File

@@ -0,0 +1,20 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
var sharedDeflateReaderPool *ReaderPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedDeflateReaderPool = NewReaderPool(maxSize, func(reader io.Reader) (Reader, error) {
return newDeflateReader(reader)
})
}

View File

@@ -0,0 +1,20 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
var sharedGzipReaderPool *ReaderPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedGzipReaderPool = NewReaderPool(maxSize, func(reader io.Reader) (Reader, error) {
return newGzipReader(reader)
})
}

View File

@@ -18,6 +18,11 @@ const (
var ErrNotSupportedContentEncoding = errors.New("not supported content encoding")
// AllEncodings 当前支持的所有编码
func AllEncodings() []ContentEncoding {
return []ContentEncoding{ContentEncodingBr, ContentEncodingGzip, ContentEncodingDeflate}
}
// NewReader 获取Reader
func NewReader(reader io.Reader, contentEncoding ContentEncoding) (Reader, error) {
switch contentEncoding {
@@ -32,7 +37,6 @@ func NewReader(reader io.Reader, contentEncoding ContentEncoding) (Reader, error
}
// NewWriter 获取Writer
// TODO 考虑重用Writer
func NewWriter(writer io.Writer, compressType serverconfigs.HTTPCompressionType, level int) (Writer, error) {
switch compressType {
case serverconfigs.HTTPCompressionTypeGzip:

View File

@@ -2,9 +2,16 @@
package compressions
import "io"
type Writer interface {
Write(p []byte) (int, error)
Flush() error
Reset(writer io.Writer)
RawClose() error
Close() error
Level() int
SetPool(pool *WriterPool)
ResetFinish()
}

View File

@@ -0,0 +1,26 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
type BaseWriter struct {
pool *WriterPool
isFinished bool
}
func (this *BaseWriter) SetPool(pool *WriterPool) {
this.pool = pool
}
func (this *BaseWriter) Finish(obj Writer) error {
err := obj.RawClose()
if err == nil && this.pool != nil && !this.isFinished {
this.pool.Put(obj)
}
this.isFinished = true
return err
}
func (this *BaseWriter) ResetFinish() {
this.isFinished = false
}

View File

@@ -8,19 +8,28 @@ import (
)
type BrotliWriter struct {
BaseWriter
writer *brotli.Writer
level int
}
func NewBrotliWriter(writer io.Writer, level int) (Writer, error) {
return sharedBrotliWriterPool.Get(writer, level)
}
func newBrotliWriter(writer io.Writer, level int) (*BrotliWriter, error) {
if level <= 0 {
level = brotli.BestSpeed
} else if level > brotli.BestCompression {
level = brotli.BestCompression
}
return &BrotliWriter{
writer: brotli.NewWriterLevel(writer, level),
level: level,
writer: brotli.NewWriterOptions(writer, brotli.WriterOptions{
Quality: level,
LGWin: 13, // TODO 在全局设置里可以设置此值
}),
level: level,
}, nil
}
@@ -32,10 +41,18 @@ func (this *BrotliWriter) Flush() error {
return this.writer.Flush()
}
func (this *BrotliWriter) Close() error {
func (this *BrotliWriter) Reset(newWriter io.Writer) {
this.writer.Reset(newWriter)
}
func (this *BrotliWriter) RawClose() error {
return this.writer.Close()
}
func (this *BrotliWriter) Close() error {
return this.Finish(this)
}
func (this *BrotliWriter) Level() int {
return this.level
}

View File

@@ -0,0 +1,116 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"strings"
"testing"
)
func BenchmarkBrotliWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
for i := 0; i < b.N; i++ {
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
}
func BenchmarkBrotliWriter_Write_Parallel(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
})
}
func BenchmarkBrotliWriter_Write_Small(b *testing.B) {
var data = []byte(strings.Repeat("A", 16))
for i := 0; i < b.N; i++ {
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
}
func BenchmarkBrotliWriter_Write_Large(b *testing.B) {
var data = []byte(strings.Repeat("A", 4096))
for i := 0; i < b.N; i++ {
var buf = &bytes.Buffer{}
writer, err := compressions.NewBrotliWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
}

View File

@@ -8,11 +8,17 @@ import (
)
type DeflateWriter struct {
BaseWriter
writer *flate.Writer
level int
}
func NewDeflateWriter(writer io.Writer, level int) (Writer, error) {
return sharedDeflateWriterPool.Get(writer, level)
}
func newDeflateWriter(writer io.Writer, level int) (Writer, error) {
if level <= 0 {
level = flate.BestSpeed
} else if level > flate.BestCompression {
@@ -38,10 +44,18 @@ func (this *DeflateWriter) Flush() error {
return this.writer.Flush()
}
func (this *DeflateWriter) Close() error {
func (this *DeflateWriter) Reset(writer io.Writer) {
this.writer.Reset(writer)
}
func (this *DeflateWriter) RawClose() error {
return this.writer.Close()
}
func (this *DeflateWriter) Close() error {
return this.Finish(this)
}
func (this *DeflateWriter) Level() int {
return this.level
}

View File

@@ -0,0 +1,36 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"strings"
"testing"
)
func BenchmarkDeflateWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
for i := 0; i < b.N; i++ {
var buf = &bytes.Buffer{}
writer, err := compressions.NewDeflateWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
}

View File

@@ -1,51 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"bytes"
"io"
)
type EncodingWriter struct {
contentEncoding ContentEncoding
writer Writer
buf *bytes.Buffer
}
func NewEncodingWriter(contentEncoding ContentEncoding, writer Writer) (Writer, error) {
return &EncodingWriter{
contentEncoding: contentEncoding,
writer: writer,
buf: &bytes.Buffer{},
}, nil
}
func (this *EncodingWriter) Write(p []byte) (int, error) {
return this.buf.Write(p)
}
func (this *EncodingWriter) Flush() error {
return this.writer.Flush()
}
func (this *EncodingWriter) Close() error {
reader, err := NewReader(this.buf, this.contentEncoding)
if err != nil {
_ = this.writer.Close()
return err
}
_, err = io.Copy(this.writer, reader)
if err != nil {
_ = reader.Close()
_ = this.writer.Close()
return err
}
_ = reader.Close()
return this.writer.Close()
}
func (this *EncodingWriter) Level() int {
return this.writer.Level()
}

View File

@@ -1,47 +0,0 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"testing"
)
func TestNewEncodingWriter(t *testing.T) {
var buf = &bytes.Buffer{}
subWriter, err := NewWriter(buf, serverconfigs.HTTPCompressionTypeGzip, 5)
if err != nil {
t.Fatal(err)
}
writer, err := NewEncodingWriter(ContentEncodingGzip, subWriter)
if err != nil {
t.Fatal(err)
}
gzipBuf := &bytes.Buffer{}
gzipWriter, err := NewGzipWriter(gzipBuf, 5)
if err != nil {
t.Fatal(err)
}
_, err = gzipWriter.Write([]byte("Hello"))
if err != nil {
t.Fatal(err)
}
_, err = gzipWriter.Write([]byte("World"))
if err != nil {
t.Fatal(err)
}
_ = gzipWriter.Close()
_, err = writer.Write(gzipBuf.Bytes())
if err != nil {
t.Fatal(err)
}
_ = writer.Close()
t.Log(buf.String())
}

View File

@@ -8,11 +8,17 @@ import (
)
type GzipWriter struct {
BaseWriter
writer *gzip.Writer
level int
}
func NewGzipWriter(writer io.Writer, level int) (Writer, error) {
return sharedGzipWriterPool.Get(writer, level)
}
func newGzipWriter(writer io.Writer, level int) (Writer, error) {
if level <= 0 {
level = gzip.BestSpeed
} else if level > gzip.BestCompression {
@@ -38,10 +44,18 @@ func (this *GzipWriter) Flush() error {
return this.writer.Flush()
}
func (this *GzipWriter) Close() error {
func (this *GzipWriter) Reset(writer io.Writer) {
this.writer.Reset(writer)
}
func (this *GzipWriter) RawClose() error {
return this.writer.Close()
}
func (this *GzipWriter) Close() error {
return this.Finish(this)
}
func (this *GzipWriter) Level() int {
return this.level
}

View File

@@ -0,0 +1,36 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions_test
import (
"bytes"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"strings"
"testing"
)
func BenchmarkGzipWriter_Write(b *testing.B) {
var data = []byte(strings.Repeat("A", 1024))
for i := 0; i < b.N; i++ {
var buf = &bytes.Buffer{}
writer, err := compressions.NewGzipWriter(buf, 5)
if err != nil {
b.Fatal(err)
}
for j := 0; j < 100; j++ {
_, err = writer.Write(data)
if err != nil {
b.Fatal(err)
}
/**err = writer.Flush()
if err != nil {
b.Fatal(err)
}**/
}
_ = writer.Close()
}
}

View File

@@ -0,0 +1,61 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"io"
)
type WriterPool struct {
m map[int]chan Writer // level => chan Writer
newFunc func(writer io.Writer, level int) (Writer, error)
}
func NewWriterPool(maxSize int, maxLevel int, newFunc func(writer io.Writer, level int) (Writer, error)) *WriterPool {
if maxSize <= 0 {
maxSize = 1024
}
var m = map[int]chan Writer{}
for i := 0; i <= maxLevel; i++ {
m[i] = make(chan Writer, maxSize)
}
return &WriterPool{
m: m,
newFunc: newFunc,
}
}
func (this *WriterPool) Get(parentWriter io.Writer, level int) (Writer, error) {
c, ok := this.m[level]
if !ok {
c = this.m[0]
}
select {
case writer := <-c:
writer.Reset(parentWriter)
writer.ResetFinish()
return writer, nil
default:
writer, err := this.newFunc(parentWriter, level)
if err != nil {
return nil, err
}
writer.SetPool(this)
return writer, nil
}
}
func (this *WriterPool) Put(writer Writer) {
var level = writer.Level()
c, ok := this.m[level]
if !ok {
c = this.m[0]
}
select {
case c <- writer:
default:
}
}

View File

@@ -0,0 +1,21 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/andybalholm/brotli"
"io"
)
var sharedBrotliWriterPool *WriterPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedBrotliWriterPool = NewWriterPool(maxSize, brotli.BestCompression, func(writer io.Writer, level int) (Writer, error) {
return newBrotliWriter(writer, level)
})
}

View File

@@ -0,0 +1,21 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"compress/flate"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
var sharedDeflateWriterPool *WriterPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedDeflateWriterPool = NewWriterPool(maxSize, flate.BestCompression, func(writer io.Writer, level int) (Writer, error) {
return newDeflateWriter(writer, level)
})
}

View File

@@ -0,0 +1,21 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package compressions
import (
"compress/gzip"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"io"
)
var sharedGzipWriterPool *WriterPool
func init() {
var maxSize = utils.SystemMemoryGB() * 256
if maxSize == 0 {
maxSize = 256
}
sharedGzipWriterPool = NewWriterPool(maxSize, gzip.BestCompression, func(writer io.Writer, level int) (Writer, error) {
return newGzipWriter(writer, level)
})
}

View File

@@ -1,12 +1,12 @@
package configs
import (
"github.com/go-yaml/yaml"
"github.com/iwind/TeaGo/Tea"
"gopkg.in/yaml.v3"
"io/ioutil"
)
// 节点API配置
// APIConfig 节点API配置
type APIConfig struct {
RPC struct {
Endpoints []string `yaml:"endpoints"`

View File

@@ -1,7 +1,7 @@
package teaconst
const (
Version = "0.4.2"
Version = "0.4.5"
ProductName = "Edge Node"
ProcessName = "edge-node"

View File

@@ -14,4 +14,7 @@ var (
NodeIdString = ""
GlobalProductName = nodeconfigs.DefaultProductName
IsQuiting = false // 是否正在退出
EnableDBStat = false // 是否开启本地数据库统计
)

View File

@@ -3,8 +3,9 @@ package events
type Event = string
const (
EventStart Event = "start" // start loading
EventLoaded Event = "loaded" // first load
EventQuit Event = "quit" // quit node gracefully
EventReload Event = "reload" // reload config
EventStart Event = "start" // start loading
EventLoaded Event = "loaded" // first load
EventQuit Event = "quit" // quit node gracefully
EventReload Event = "reload" // reload config
EventTerminated Event = "terminated" // process terminated
)

View File

@@ -1,6 +1,7 @@
package events
import (
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"sync"
)
@@ -59,6 +60,14 @@ func Remove(key interface{}) {
// Notify 通知事件
func Notify(event Event) {
// 特殊事件
switch event {
case EventQuit:
teaconst.IsQuiting = true
case EventTerminated:
teaconst.IsQuiting = true
}
locker.Lock()
m := eventsMap[event]
locker.Unlock()

View File

@@ -62,29 +62,39 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if err != nil {
return err
}
{
cmd := exec.Command(path, "create", this.config.WhiteName, "hash:ip", "timeout", "0", "maxelem", "1000000")
stderr := bytes.NewBuffer([]byte{})
// ipv4
for _, listName := range []string{this.config.WhiteName, this.config.BlackName} {
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "create", listName, "hash:ip", "timeout", "0", "maxelem", "1000000")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
var output = stderr.Bytes()
if !bytes.Contains(output, []byte("already exists")) {
return errors.New("create ipset '" + this.config.WhiteName + "': " + err.Error() + ", output: " + string(output))
return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
} else {
err = nil
}
}
}
{
cmd := exec.Command(path, "create", this.config.BlackName, "hash:ip", "timeout", "0", "maxelem", "1000000")
stderr := bytes.NewBuffer([]byte{})
// ipv6
for _, listName := range []string{this.config.WhiteNameIPv6, this.config.BlackNameIPv6} {
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "create", listName, "hash:ip", "family", "inet6", "timeout", "0", "maxelem", "1000000")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
var output = stderr.Bytes()
if !bytes.Contains(output, []byte("already exists")) {
return errors.New("create ipset '" + this.config.BlackName + "': " + err.Error() + ", output: " + string(output))
return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
} else {
err = nil
}
@@ -99,8 +109,12 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
return err
}
{
cmd := exec.Command(path, "--permanent", "--new-ipset="+this.config.WhiteName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
// ipv4
for _, listName := range []string{this.config.WhiteName, this.config.BlackName} {
if len(listName) == 0 {
continue
}
cmd := exec.Command(path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
@@ -109,45 +123,57 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
if bytes.Contains(output, []byte("NAME_CONFLICT")) {
err = nil
} else {
return errors.New("firewall-cmd add ipset '" + this.config.WhiteName + "': " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
}
}
}
{
cmd := exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+this.config.WhiteName+"' accept")
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
return errors.New("firewall-cmd add rich rule '" + this.config.WhiteName + "': " + err.Error() + ", output: " + string(output))
// ipv6
for _, listName := range []string{this.config.WhiteNameIPv6, this.config.BlackNameIPv6} {
if len(listName) == 0 {
continue
}
}
{
cmd := exec.Command(path, "--permanent", "--new-ipset="+this.config.BlackName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
cmd := exec.Command(path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=family=inet6", "--option=timeout=0", "--option=maxelem=1000000")
stderr := bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
var output = stderr.Bytes()
if bytes.Contains(output, []byte("NAME_CONFLICT")) {
err = nil
} else {
return errors.New("firewall-cmd add ipset '" + this.config.BlackName + "': " + err.Error() + ", output: " + string(output))
return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
}
}
}
{
cmd := exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+this.config.BlackName+"' reject")
stderr := bytes.NewBuffer([]byte{})
// accept
for _, listName := range []string{this.config.WhiteName, this.config.WhiteNameIPv6} {
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' accept")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
return errors.New("firewall-cmd add rich rule '" + this.config.WhiteName + "': " + err.Error() + ", output: " + string(output))
var output = stderr.Bytes()
return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + string(output))
}
}
// reject
for _, listName := range []string{this.config.BlackName, this.config.BlackNameIPv6} {
if len(listName) == 0 {
continue
}
var cmd = exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' reject")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
var output = stderr.Bytes()
return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + string(output))
}
}
@@ -158,7 +184,7 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
var output = stderr.Bytes()
return errors.New("firewall-cmd reload: " + err.Error() + ", output: " + string(output))
}
}
@@ -171,38 +197,48 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
return err
}
{
// accept
for _, listName := range []string{this.config.WhiteName, this.config.WhiteNameIPv6} {
if len(listName) == 0 {
continue
}
// 检查规则是否存在
var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", this.config.WhiteName, "src", "-j", "ACCEPT")
var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
err := cmd.Run()
var exists = err == nil
// 添加规则
if !exists {
var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", this.config.WhiteName, "src", "-j", "ACCEPT")
stderr := bytes.NewBuffer([]byte{})
var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
var output = stderr.Bytes()
return errors.New("iptables add rule: " + err.Error() + ", output: " + string(output))
}
}
}
{
// reject
for _, listName := range []string{this.config.BlackName, this.config.BlackNameIPv6} {
if len(listName) == 0 {
continue
}
// 检查规则是否存在
var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", this.config.BlackName, "src", "-j", "REJECT")
var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
err := cmd.Run()
var exists = err == nil
if !exists {
var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", this.config.BlackName, "src", "-j", "REJECT")
stderr := bytes.NewBuffer([]byte{})
var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
var stderr = bytes.NewBuffer([]byte{})
cmd.Stderr = stderr
err := cmd.Run()
if err != nil {
output := stderr.Bytes()
var output = stderr.Bytes()
return errors.New("iptables add rule: " + err.Error() + ", output: " + string(output))
}
}
@@ -236,7 +272,7 @@ func (this *IPSetAction) runAction(action string, listType IPListType, item *pb.
return nil
}
for _, cidr := range cidrList {
index := strings.Index(cidr, "/")
var index = strings.Index(cidr, "/")
if index <= 0 {
continue
}
@@ -256,26 +292,40 @@ func (this *IPSetAction) runAction(action string, listType IPListType, item *pb.
return nil
}
func (this *IPSetAction) SetConfig(config *firewallconfigs.FirewallActionIPSetConfig) {
this.config = config
}
func (this *IPSetAction) runActionSingleIP(action string, listType IPListType, item *pb.IPItem) error {
if item.Type == "all" {
return nil
}
listName := ""
var listName = ""
var isIPv6 = strings.Contains(item.IpFrom, ":")
switch listType {
case IPListTypeWhite:
listName = this.config.WhiteName
if isIPv6 {
listName = this.config.WhiteNameIPv6
} else {
listName = this.config.WhiteName
}
case IPListTypeBlack:
listName = this.config.BlackName
if isIPv6 {
listName = this.config.BlackNameIPv6
} else {
listName = this.config.BlackName
}
default:
// 不支持的类型
return nil
}
if len(listName) == 0 {
return errors.New("empty list name for '" + listType + "'")
return nil
}
path := this.config.Path
var path = this.config.Path
var err error
if len(path) == 0 {
path, err = exec.LookPath("ipset")
@@ -290,7 +340,7 @@ func (this *IPSetAction) runActionSingleIP(action string, listType IPListType, i
}
// ipset add edge_ip_list 192.168.2.32 timeout 30
args := []string{}
var args = []string{}
switch action {
case "addItem":
args = append(args, "add")
@@ -300,7 +350,7 @@ func (this *IPSetAction) runActionSingleIP(action string, listType IPListType, i
args = append(args, listName, item.IpFrom)
if action == "addItem" {
timestamp := time.Now().Unix()
var timestamp = time.Now().Unix()
if item.ExpiredAt > timestamp {
args = append(args, "timeout", strconv.FormatInt(item.ExpiredAt-timestamp, 10))
}
@@ -312,7 +362,7 @@ func (this *IPSetAction) runActionSingleIP(action string, listType IPListType, i
}
this.errorBuf.Reset()
cmd := exec.Command(path, args...)
var cmd = exec.Command(path, args...)
cmd.Stderr = this.errorBuf
err = cmd.Run()
if err != nil {

View File

@@ -1,15 +1,16 @@
package iplibrary
package iplibrary_test
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
"github.com/iwind/TeaGo/maps"
"testing"
"time"
)
func TestIPSetAction_Init(t *testing.T) {
action := NewIPSetAction()
action := iplibrary.NewIPSetAction()
err := action.Init(&firewallconfigs.FirewallActionConfig{
Params: maps.Map{
"path": "/usr/bin/iptables",
@@ -24,14 +25,16 @@ func TestIPSetAction_Init(t *testing.T) {
}
func TestIPSetAction_AddItem(t *testing.T) {
action := NewIPSetAction()
action.config = &firewallconfigs.FirewallActionIPSetConfig{
Path: "/usr/bin/iptables",
WhiteName: "white-list",
BlackName: "black-list",
}
var action = iplibrary.NewIPSetAction()
action.SetConfig(&firewallconfigs.FirewallActionIPSetConfig{
Path: "/usr/bin/iptables",
WhiteName: "white-list",
BlackName: "black-list",
WhiteNameIPv6: "white-list-ipv6",
BlackNameIPv6: "black-list-ipv6",
})
{
err := action.AddItem(IPListTypeWhite, &pb.IPItem{
err := action.AddItem(iplibrary.IPListTypeWhite, &pb.IPItem{
Type: "ipv4",
Id: 1,
IpFrom: "192.168.1.100",
@@ -42,9 +45,20 @@ func TestIPSetAction_AddItem(t *testing.T) {
}
t.Log("ok")
}
{
err := action.AddItem(IPListTypeBlack, &pb.IPItem{
err := action.AddItem(iplibrary.IPListTypeWhite, &pb.IPItem{
Type: "ipv4",
Id: 1,
IpFrom: "1:2:3:4",
ExpiredAt: time.Now().Unix() + 30,
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
{
err := action.AddItem(iplibrary.IPListTypeBlack, &pb.IPItem{
Type: "ipv4",
Id: 1,
IpFrom: "192.168.1.100",
@@ -55,10 +69,22 @@ func TestIPSetAction_AddItem(t *testing.T) {
}
t.Log("ok")
}
{
err := action.AddItem(iplibrary.IPListTypeBlack, &pb.IPItem{
Type: "ipv4",
Id: 1,
IpFrom: "1:2:3:4",
ExpiredAt: time.Now().Unix() + 30,
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
}
func TestIPSetAction_DeleteItem(t *testing.T) {
action := NewIPSetAction()
action := iplibrary.NewIPSetAction()
err := action.Init(&firewallconfigs.FirewallActionConfig{
Params: maps.Map{
"path": "/usr/bin/firewalld",
@@ -68,7 +94,7 @@ func TestIPSetAction_DeleteItem(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = action.DeleteItem(IPListTypeWhite, &pb.IPItem{
err = action.DeleteItem(iplibrary.IPListTypeWhite, &pb.IPItem{
Type: "ipv4",
Id: 1,
IpFrom: "192.168.1.100",

View File

@@ -169,6 +169,10 @@ func (this *IPList) sortItems() {
// 不加锁的情况下查找Item
func (this *IPList) lookupIP(ip uint64) *IPItem {
if len(this.sortedItems) == 0 {
return nil
}
var count = len(this.sortedItems)
var resultIndex = -1
sort.Search(count, func(i int) bool {

View File

@@ -5,20 +5,27 @@ package iplibrary
import (
"database/sql"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/iwind/TeaGo/Tea"
_ "github.com/mattn/go-sqlite3"
"os"
"path/filepath"
"time"
)
type IPListDB struct {
db *sql.DB
itemTableName string
deleteItemStmt *sql.Stmt
insertItemStmt *sql.Stmt
selectItemsStmt *sql.Stmt
itemTableName string
deleteExpiredItemsStmt *sql.Stmt
deleteItemStmt *sql.Stmt
insertItemStmt *sql.Stmt
selectItemsStmt *sql.Stmt
selectMaxVersionStmt *sql.Stmt
cleanTicker *time.Ticker
dir string
}
@@ -27,6 +34,7 @@ func NewIPListDB() (*IPListDB, error) {
var db = &IPListDB{
itemTableName: "ipItems",
dir: filepath.Clean(Tea.Root + "/data"),
cleanTicker: time.NewTicker(24 * time.Hour),
}
err := db.init()
return db, err
@@ -43,7 +51,7 @@ func (this *IPListDB) init() error {
remotelogs.Println("CACHE", "create cache dir '"+this.dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL")
db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
@@ -83,6 +91,11 @@ ON "` + this.itemTableName + `" (
}
// 初始化SQL语句
this.deleteExpiredItemsStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemTableName + `" WHERE "expiredAt">0 AND "expiredAt"<?`)
if err != nil {
return err
}
this.deleteItemStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemTableName + `" WHERE "itemId"=?`)
if err != nil {
return err
@@ -93,16 +106,37 @@ ON "` + this.itemTableName + `" (
return err
}
this.selectItemsStmt, err = this.db.Prepare(`SELECT "listId", "listType", "isGlobal", "type", "itemId", "ipFrom", "ipTo", "expiredAt", "eventLevel", "isDeleted", "version", "nodeId", "serverId" FROM "` + this.itemTableName + `" ORDER BY "version" ASC, "itemId" ASC LIMIT ?, ?`)
this.selectItemsStmt, err = this.db.Prepare(`SELECT "listId", "listType", "isGlobal", "type", "itemId", "ipFrom", "ipTo", "expiredAt", "eventLevel", "isDeleted", "version", "nodeId", "serverId" FROM "` + this.itemTableName + `" WHERE isDeleted=0 ORDER BY "version" ASC, "itemId" ASC LIMIT ?, ?`)
if err != nil {
return err
}
this.selectMaxVersionStmt, err = this.db.Prepare(`SELECT "version" FROM "` + this.itemTableName + `" ORDER BY "id" DESC LIMIT 1`)
this.db = db
goman.New(func() {
events.On(events.EventQuit, func() {
this.cleanTicker.Stop()
})
for range this.cleanTicker.C {
err := this.DeleteExpiredItems()
if err != nil {
remotelogs.Error("IP_LIST_DB", "clean expired items failed: "+err.Error())
}
}
})
return nil
}
// DeleteExpiredItems 删除过期的条目
func (this *IPListDB) DeleteExpiredItems() error {
_, err := this.deleteExpiredItemsStmt.Exec(time.Now().Unix() - 7*86400)
return err
}
func (this *IPListDB) AddItem(item *pb.IPItem) error {
_, err := this.deleteItemStmt.Exec(item.Id)
if err != nil {
@@ -133,11 +167,27 @@ func (this *IPListDB) ReadItems(offset int64, size int64) (items []*pb.IPItem, e
return
}
// ReadMaxVersion 读取当前最大版本号
func (this *IPListDB) ReadMaxVersion() int64 {
row := this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err = row.Scan(&version)
if err != nil {
return 0
}
return version
}
func (this *IPListDB) Close() error {
if this.db != nil {
_ = this.deleteExpiredItemsStmt.Close()
_ = this.deleteItemStmt.Close()
_ = this.insertItemStmt.Close()
_ = this.selectItemsStmt.Close()
_ = this.selectMaxVersionStmt.Close()
return this.db.Close()
}

View File

@@ -1,9 +1,10 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package iplibrary
package iplibrary_test
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
_ "github.com/iwind/TeaGo/bootstrap"
"github.com/iwind/TeaGo/logs"
"testing"
@@ -11,7 +12,7 @@ import (
)
func TestIPListDB_AddItem(t *testing.T) {
db, err := NewIPListDB()
db, err := iplibrary.NewIPListDB()
if err != nil {
t.Fatal(err)
}
@@ -48,7 +49,7 @@ func TestIPListDB_AddItem(t *testing.T) {
}
func TestIPListDB_ReadItems(t *testing.T) {
db, err := NewIPListDB()
db, err := iplibrary.NewIPListDB()
if err != nil {
t.Fatal(err)
}
@@ -58,3 +59,11 @@ func TestIPListDB_ReadItems(t *testing.T) {
}
logs.PrintAsJSON(items, t)
}
func TestIPListDB_ReadMaxVersion(t *testing.T) {
db, err := iplibrary.NewIPListDB()
if err != nil {
t.Fatal(err)
}
t.Log(db.ReadMaxVersion())
}

View File

@@ -302,16 +302,20 @@ func TestTooManyLists(t *testing.T) {
func BenchmarkIPList_Contains(b *testing.B) {
runtime.GOMAXPROCS(1)
list := NewIPList()
for i := 1; i < 194; i++ {
list.Add(&IPItem{
var list = NewIPList()
for i := 1; i < 200_000; i++ {
list.AddDelay(&IPItem{
Id: int64(i),
IPFrom: utils.IP2Long(strconv.Itoa(i%255) + "." + strconv.Itoa(i%255) + ".0.1"),
IPTo: utils.IP2Long(strconv.Itoa(i%255) + "." + strconv.Itoa(i%255) + ".0.1"),
IPFrom: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
IPTo: utils.IP2Long(strconv.Itoa(rands.Int(0, 255)) + "." + strconv.Itoa(rands.Int(0, 255)) + ".0.1"),
ExpiredAt: time.Now().Unix() + 60,
})
}
list.Sort()
b.Log(len(list.itemsMap), "ip")
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = list.Contains(utils.IP2Long("192.168.1.100"))
}

View File

@@ -8,37 +8,37 @@ import (
// AllowIP 检查IP是否被允许访问
// 如果一个IP不在任何名单中则允许访问
func AllowIP(ip string, serverId int64) bool {
func AllowIP(ip string, serverId int64) (canGoNext bool, inAllowList bool) {
var ipLong = utils.IP2Long(ip)
if ipLong == 0 {
return false
return false, false
}
// check white lists
if GlobalWhiteIPList.Contains(ipLong) {
return true
return true, true
}
if serverId > 0 {
var list = SharedServerListManager.FindWhiteList(serverId, false)
if list != nil && list.Contains(ipLong) {
return true
return true, true
}
}
// check black lists
if GlobalBlackIPList.Contains(ipLong) {
return false
return false, false
}
if serverId > 0 {
var list = SharedServerListManager.FindBlackList(serverId, false)
if list != nil && list.Contains(ipLong) {
return false
return false, false
}
}
return true
return true, false
}
// IsInWhiteList 检查IP是否在白名单中
@@ -58,7 +58,7 @@ func AllowIPStrings(ipStrings []string, serverId int64) bool {
return true
}
for _, ip := range ipStrings {
isAllowed := AllowIP(ip, serverId)
isAllowed, _ := AllowIP(ip, serverId)
if !isAllowed {
return false
}

View File

@@ -61,7 +61,7 @@ func (this *IPListManager) Start() {
if Tea.IsTesting() {
this.ticker = time.NewTicker(10 * time.Second)
}
countErrors := 0
var countErrors = 0
for {
select {
case <-this.ticker.C:
@@ -100,6 +100,13 @@ func (this *IPListManager) init() {
} else {
this.db = db
// 删除本地数据库中过期的条目
_ = db.DeleteExpiredItems()
// 本地数据库中最大版本号
this.version = db.ReadMaxVersion()
// 从本地数据库中加载
var offset int64 = 0
var size int64 = 1000
for {
@@ -171,7 +178,7 @@ func (this *IPListManager) FindList(listId int64) *IPList {
return list
}
func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool) {
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
this.locker.Lock()
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
@@ -205,10 +212,10 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool)
list.Delete(item.Id)
// 从WAF名单中删除
waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId, shouldExecute)
waf.SharedIPBlackList.RemoveIP(item.IpFrom, item.ServerId, fromRemote)
// 操作事件
if shouldExecute {
if fromRemote {
SharedActionManager.DeleteItem(item.ListType, item)
}
@@ -225,7 +232,7 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool)
})
// 事件操作
if shouldExecute {
if fromRemote {
SharedActionManager.DeleteItem(item.ListType, item)
SharedActionManager.AddItem(item.ListType, item)
}
@@ -236,5 +243,11 @@ func (this *IPListManager) processItems(items []*pb.IPItem, shouldExecute bool)
}
this.locker.Unlock()
this.version = items[len(items)-1].Version
if fromRemote {
var latestVersion = items[len(items)-1].Version
if latestVersion > this.version {
this.version = latestVersion
}
}
}

View File

@@ -7,17 +7,18 @@ import (
"encoding/json"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/iwind/TeaGo/Tea"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
"strings"
"sync"
"time"
)
@@ -38,7 +39,7 @@ type Task struct {
item *serverconfigs.MetricItemConfig
isLoaded bool
db *sql.DB
db *dbs.DB
statTableName string
isStopped bool
@@ -87,12 +88,16 @@ func (this *Task) Init() error {
remotelogs.Println("METRIC", "create data dir '"+dir+"'")
}
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL")
db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
if err != nil {
return err
}
db.SetMaxOpenConns(1)
this.db = db
this.db = dbs.NewDB(db)
if teaconst.EnableDBStat {
this.db.EnableStat(true)
}
//创建统计表
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.statTableName + `" (
@@ -438,10 +443,11 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
if len(idStrings) > 0 {
// 设置为已上传
_, err = this.db.Exec(`UPDATE "` + this.statTableName + `" SET isUploaded=1 WHERE id IN (` + strings.Join(idStrings, ",") + `)`)
// TODO 先不判断是否已经上传需要改造API进行配合
/**_, err = this.db.Exec(`UPDATE "` + this.statTableName + `" SET isUploaded=1 WHERE id IN (` + strings.Join(idStrings, ",") + `)`)
if err != nil {
return err
}
}**/
}
}

View File

@@ -9,6 +9,7 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/configs"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/errors"
@@ -182,7 +183,7 @@ func (this *APIStream) handleWriteCache(message *pb.NodeStreamMessage) error {
}
expiredAt := time.Now().Unix() + msg.LifeSeconds
writer, err := storage.OpenWriter(msg.Key, expiredAt, 200, int64(len(msg.Value)), false)
writer, err := storage.OpenWriter(msg.Key, expiredAt, 200, int64(len(msg.Value)), -1, false)
if err != nil {
this.replyFail(message.RequestId, "prepare writing failed: "+err.Error())
return err
@@ -239,7 +240,7 @@ func (this *APIStream) handleReadCache(message *pb.NodeStreamMessage) error {
}()
}
reader, err := storage.OpenReader(msg.Key, false)
reader, err := storage.OpenReader(msg.Key, false, false)
if err != nil {
if err == caches.ErrNotFound {
this.replyFail(message.RequestId, "key not found")
@@ -350,7 +351,16 @@ func (this *APIStream) handlePurgeCache(message *pb.NodeStreamMessage) error {
if msg.Type == "file" {
var keys = msg.Keys
for _, key := range keys {
keys = append(keys, key+webpSuffix)
keys = append(keys,
key+caches.SuffixMethod+"HEAD",
key+caches.SuffixWebP,
key+caches.SuffixPartial,
)
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
keys = append(keys, key+caches.SuffixCompression+encoding)
keys = append(keys, key+caches.SuffixWebP+caches.SuffixCompression+encoding)
}
}
msg.Keys = keys
}
@@ -462,7 +472,7 @@ func (this *APIStream) handlePreheatCache(message *pb.NodeStreamMessage) error {
}
expiredAt := time.Now().Unix() + 8600
writer, err := storage.OpenWriter(key, expiredAt, 200, resp.ContentLength, false) // TODO 可以设置缓存过期时间
writer, err := storage.OpenWriter(key, expiredAt, 200, resp.ContentLength, -1, false) // TODO 可以设置缓存过期时间
if err != nil {
locker.Lock()
errorMessages = append(errorMessages, "open cache writer failed: "+key+": "+err.Error())

View File

@@ -49,8 +49,11 @@ func (this *ClientListener) Accept() (net.Conn, error) {
// 是否在WAF名单中
ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
if err == nil {
if !iplibrary.AllowIP(ip, 0) || (!waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)) {
canGoNext, _ := iplibrary.AllowIP(ip, 0)
if !canGoNext ||
(!waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)) {
tcpConn, ok := conn.(*net.TCPConn)
if ok {
_ = tcpConn.SetLinger(0)

View File

@@ -94,7 +94,7 @@ Loop:
// 是否包含了invalid UTF-8
if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
for _, accessLog := range accessLogs {
this.toValidUTF8(accessLog)
this.ToValidUTF8(accessLog)
}
// 重新提交
@@ -108,7 +108,7 @@ Loop:
return nil
}
func (this *HTTPAccessLogQueue) toValidUTF8(accessLog *pb.HTTPAccessLog) {
func (this *HTTPAccessLogQueue) ToValidUTF8(accessLog *pb.HTTPAccessLog) {
accessLog.RemoteUser = utils.ToValidUTF8string(accessLog.RemoteUser)
accessLog.RequestURI = utils.ToValidUTF8string(accessLog.RequestURI)
accessLog.RequestPath = utils.ToValidUTF8string(accessLog.RequestPath)

View File

@@ -1,15 +1,18 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
package nodes_test
import (
"bytes"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/nodes"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
_ "github.com/iwind/TeaGo/bootstrap"
"google.golang.org/grpc/status"
"reflect"
"runtime"
"runtime/debug"
"strconv"
"strings"
"testing"
@@ -47,7 +50,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
},
}
new(HTTPAccessLogQueue).toValidUTF8(accessLog)
new(nodes.HTTPAccessLogQueue).ToValidUTF8(accessLog)
// logs.PrintAsJSON(accessLog)
@@ -105,6 +108,33 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
t.Log("ok")
}
func TestHTTPAccessLogQueue_Memory(t *testing.T) {
testutils.StartMemoryStats(t)
debug.SetGCPercent(10)
var accessLogs = []*pb.HTTPAccessLog{}
for i := 0; i < 20_000; i++ {
accessLogs = append(accessLogs, &pb.HTTPAccessLog{
RequestPath: "https://goedge.cn/hello/world",
})
}
runtime.GC()
// will not release automatically
func() {
var accessLogs1 = []*pb.HTTPAccessLog{}
for i := 0; i < 2_000_000; i++ {
accessLogs1 = append(accessLogs1, &pb.HTTPAccessLog{
RequestPath: "https://goedge.cn/hello/world",
})
}
}()
time.Sleep(5 * time.Second)
}
func BenchmarkHTTPAccessLogQueue_ToValidUTF8(b *testing.B) {
runtime.GOMAXPROCS(1)

View File

@@ -42,7 +42,7 @@ func NewHTTPClientPool() *HTTPClientPool {
}
// Client 根据地址获取客户端
func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.OriginConfig, originAddr string, proxyProtocol *serverconfigs.ProxyProtocolConfig) (rawClient *http.Client, err error) {
func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.OriginConfig, originAddr string, proxyProtocol *serverconfigs.ProxyProtocolConfig, followRedirects bool) (rawClient *http.Client, err error) {
if origin.Addr == nil {
return nil, errors.New("origin addr should not be empty (originId:" + strconv.FormatInt(origin.Id, 10) + ")")
}
@@ -139,6 +139,19 @@ func (this *HTTPClientPool) Client(req *HTTPRequest, origin *serverconfigs.Origi
Timeout: readTimeout,
Transport: transport,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if followRedirects {
var schemeIsSame = true
for _, r := range via {
if r.URL.Scheme != req.URL.Scheme {
schemeIsSame = false
break
}
}
if schemeIsSame {
return nil
}
}
return http.ErrUseLastResponse
},
}

View File

@@ -21,14 +21,14 @@ func TestHTTPClientPool_Client(t *testing.T) {
t.Fatal(err)
}
{
client, err := pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
client, err := pool.Client(nil, origin, origin.Addr.PickAddress(), nil, false)
if err != nil {
t.Fatal(err)
}
t.Log("client:", client)
}
for i := 0; i < 10; i++ {
client, err := pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
client, err := pool.Client(nil, origin, origin.Addr.PickAddress(), nil, false)
if err != nil {
t.Fatal(err)
}
@@ -53,7 +53,7 @@ func TestHTTPClientPool_cleanClients(t *testing.T) {
for i := 0; i < 10; i++ {
t.Log("get", i)
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil, false)
time.Sleep(1 * time.Second)
}
}
@@ -73,6 +73,6 @@ func BenchmarkHTTPClientPool_Client(b *testing.B) {
pool := NewHTTPClientPool()
for i := 0; i < b.N; i++ {
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil)
_, _ = pool.Client(nil, origin, origin.Addr.PickAddress(), nil, false)
}
}

View File

@@ -5,10 +5,13 @@ import (
"errors"
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/caches"
"github.com/TeaOSLab/EdgeNode/internal/compressions"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/TeaOSLab/EdgeNode/internal/utils"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
"io"
"net/http"
"path/filepath"
@@ -35,6 +38,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return
}
// 添加 X-Cache Header
var addStatusHeader = this.web.Cache.AddStatusHeader
if addStatusHeader {
defer func() {
@@ -62,7 +66,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
break
}
}
if this.cacheRef == nil {
if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
// 检查策略默认的缓存条件
for _, cacheRef := range cachePolicy.CacheRefs {
if !cacheRef.IsOn ||
@@ -79,10 +83,10 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
break
}
}
}
if this.cacheRef == nil {
return
}
if this.cacheRef == nil {
return
}
// 校验请求
@@ -106,12 +110,20 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
// TODO 支持Vary Header
// 缓存标签
var tags = []string{}
// 检查是否有缓存
key := this.Format(this.cacheRef.Key)
var key = this.Format(this.cacheRef.Key)
if len(key) == 0 {
this.cacheRef = nil
return
}
var method = this.Method()
if method != http.MethodGet {
key += caches.SuffixMethod + method
tags = append(tags, strings.ToLower(method))
}
this.cacheKey = key
this.varMapping["cache.key"] = key
@@ -122,16 +134,32 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
this.cacheRef = nil
return
}
this.writer.cacheStorage = storage
// 判断是否在Purge
if this.web.Cache.PurgeIsOn && strings.ToUpper(this.RawReq.Method) == "PURGE" && this.RawReq.Header.Get("X-Edge-Purge-Key") == this.web.Cache.PurgeKey {
this.varMapping["cache.status"] = "PURGE"
err := storage.Delete(key)
if err != nil {
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
var subKeys = []string{
key,
key + caches.SuffixMethod + "HEAD",
key + caches.SuffixWebP,
key + caches.SuffixPartial,
}
// TODO 根据实际缓存的内容进行组合
for _, encoding := range compressions.AllEncodings() {
subKeys = append(subKeys, key+caches.SuffixCompression+encoding)
subKeys = append(subKeys, key+caches.SuffixWebP+caches.SuffixCompression+encoding)
}
for _, subKey := range subKeys {
err := storage.Delete(subKey)
if err != nil {
remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
}
}
// 通过API节点清除别节点上的的Key
// TODO 改为队列不需要每个请求都使用goroutine
goman.New(func() {
rpcClient, err := rpc.SharedRPC()
if err == nil {
@@ -160,21 +188,72 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var reader caches.Reader
var err error
// 是否优先检查WebP
var isWebP = false
if this.web.WebP != nil &&
var rangeHeader = this.RawReq.Header.Get("Range")
var isPartialRequest = len(rangeHeader) > 0
// 检查是否支持WebP
var webPIsEnabled = false
var isHeadMethod = method == http.MethodHead
if !isPartialRequest &&
!isHeadMethod &&
this.web.WebP != nil &&
this.web.WebP.IsOn &&
this.web.WebP.MatchRequest(filepath.Ext(this.Path()), this.Format) &&
this.web.WebP.MatchAccept(this.requestHeader("Accept")) {
reader, _ = storage.OpenReader(key+webpSuffix, useStale)
this.web.WebP.MatchAccept(this.RawReq.Header.Get("Accept")) {
webPIsEnabled = true
}
// 检查压缩缓存
if !isPartialRequest && !isHeadMethod && reader == nil {
if this.web.Compression != nil && this.web.Compression.IsOn {
_, encoding, ok := this.web.Compression.MatchAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"))
if ok {
// 检查支持WebP的压缩缓存
if webPIsEnabled {
reader, _ = storage.OpenReader(key+caches.SuffixWebP+caches.SuffixCompression+encoding, useStale, false)
if reader != nil {
tags = append(tags, "webp", encoding)
}
}
// 检查普通压缩缓存
if reader == nil {
reader, _ = storage.OpenReader(key+caches.SuffixCompression+encoding, useStale, false)
if reader != nil {
tags = append(tags, encoding)
}
}
}
}
}
// 检查WebP
if !isPartialRequest &&
!isHeadMethod &&
reader == nil &&
webPIsEnabled {
reader, _ = storage.OpenReader(key+caches.SuffixWebP, useStale, false)
if reader != nil {
isWebP = true
this.writer.cacheReaderSuffix = caches.SuffixWebP
tags = append(tags, "webp")
}
}
// 检查正常的文件
var isPartialCache = false
var partialRanges []rangeutils.Range
if reader == nil {
reader, err = storage.OpenReader(key, useStale)
reader, err = storage.OpenReader(key, useStale, false)
if err != nil && this.cacheRef.AllowPartialContent {
pReader, ranges := this.tryPartialReader(storage, key, useStale, rangeHeader)
if pReader != nil {
isPartialCache = true
reader = pReader
partialRanges = ranges
err = nil
}
}
if err != nil {
if err == caches.ErrNotFound {
// cache相关变量
@@ -208,7 +287,16 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 准备Buffer
var pool = this.bytePool(reader.BodySize())
var fileSize = reader.BodySize()
var totalSizeString = types.String(fileSize)
if isPartialCache {
fileSize = reader.(*caches.PartialFileReader).MaxLength()
if totalSizeString == "0" {
totalSizeString = "*"
}
}
var pool = this.bytePool(fileSize)
var buf = pool.Get()
defer func() {
pool.Put(buf)
@@ -265,13 +353,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var eTag = ""
var lastModifiedAt = reader.LastModified()
if lastModifiedAt > 0 {
if isWebP {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_webp" + "\""
if len(tags) > 0 {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "_" + strings.Join(tags, "_") + "\""
} else {
eTag = "\"" + strconv.FormatInt(lastModifiedAt, 10) + "\""
}
respHeader.Del("Etag")
respHeader["ETag"] = []string{eTag}
if !isPartialCache {
respHeader["ETag"] = []string{eTag}
}
}
// 支持 Last-Modified
@@ -279,11 +369,13 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
var modifiedTime = ""
if lastModifiedAt > 0 {
modifiedTime = time.Unix(utils.GMTUnixTime(lastModifiedAt), 0).Format("Mon, 02 Jan 2006 15:04:05") + " GMT"
respHeader.Set("Last-Modified", modifiedTime)
if !isPartialCache {
respHeader.Set("Last-Modified", modifiedTime)
}
}
// 支持 If-None-Match
if len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
if !isPartialCache && len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.writer.WriteHeader(http.StatusNotModified)
@@ -294,7 +386,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
// 支持 If-Modified-Since
if len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
if !isPartialCache && len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
// 自定义Header
this.processResponseHeaders(http.StatusNotModified)
this.writer.WriteHeader(http.StatusNotModified)
@@ -312,69 +404,56 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
this.writer.WriteHeader(reader.Status())
} else {
ifRangeHeaders, ok := this.RawReq.Header["If-Range"]
supportRange := true
var supportRange = true
if ok {
supportRange = false
for _, v := range ifRangeHeaders {
if v == this.writer.Header().Get("ETag") || v == this.writer.Header().Get("Last-Modified") {
supportRange = true
break
}
}
}
// 支持Range
rangeSet := [][]int64{}
var ranges = partialRanges
if supportRange {
fileSize := reader.BodySize()
contentRange := this.RawReq.Header.Get("Range")
if len(contentRange) > 0 {
if len(rangeHeader) > 0 {
if fileSize == 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
set, ok := httpRequestParseContentRange(contentRange)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
if len(ranges) == 0 {
ranges, ok = httpRequestParseRangeHeader(rangeHeader)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
}
if len(set) > 0 {
rangeSet = set
for _, arr := range rangeSet {
if arr[0] == -1 {
arr[0] = fileSize + arr[1]
arr[1] = fileSize - 1
if arr[0] < 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
}
if arr[1] < 0 {
arr[1] = fileSize - 1
}
if arr[1] >= fileSize {
arr[1] = fileSize - 1
}
if arr[0] > arr[1] {
if len(ranges) > 0 {
for k, r := range ranges {
r2, ok := r.Convert(fileSize)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
ranges[k] = r2
}
}
}
}
if len(rangeSet) == 1 {
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(reader.BodySize(), 10))
respHeader.Set("Content-Length", strconv.FormatInt(rangeSet[0][1]-rangeSet[0][0]+1, 10))
if len(ranges) == 1 {
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(totalSizeString))
respHeader.Set("Content-Length", strconv.FormatInt(ranges[0].Length(), 10))
this.writer.WriteHeader(http.StatusPartialContent)
err = reader.ReadBodyRange(buf, rangeSet[0][0], rangeSet[0][1], func(n int) (goNext bool, err error) {
err = reader.ReadBodyRange(buf, ranges[0].Start(), ranges[0].End(), func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, errWritingToClient
@@ -394,15 +473,15 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
return
}
} else if len(rangeSet) > 1 {
boundary := httpRequestGenBoundary()
} else if len(ranges) > 1 {
var boundary = httpRequestGenBoundary()
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
respHeader.Del("Content-Length")
contentType := respHeader.Get("Content-Type")
this.writer.WriteHeader(http.StatusPartialContent)
for index, set := range rangeSet {
for index, r := range ranges {
if index == 0 {
_, err = this.writer.WriteString("--" + boundary + "\r\n")
} else {
@@ -413,7 +492,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
return true
}
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(reader.BodySize(), 10) + "\r\n")
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(totalSizeString) + "\r\n")
if err != nil {
// 不提示写入客户端错误
return true
@@ -427,7 +506,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
}
err := reader.ReadBodyRange(buf, set[0], set[1], func(n int) (goNext bool, err error) {
err := reader.ReadBodyRange(buf, r.Start(), r.End(), func(n int) (goNext bool, err error) {
_, err = this.writer.Write(buf[:n])
if err != nil {
return false, errWritingToClient
@@ -451,7 +530,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
}
} else { // 没有Range
var resp = &http.Response{Body: reader}
this.writer.Prepare(resp, reader.BodySize(), reader.Status(), false)
this.writer.Prepare(resp, fileSize, reader.Status(), false)
this.writer.WriteHeader(reader.Status())
_, err = io.CopyBuffer(this.writer, resp.Body, buf)
@@ -492,3 +571,49 @@ func (this *HTTPRequest) addExpiresHeader(expiresAt int64) {
}
}
}
// 尝试读取区间缓存
func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key string, useStale bool, rangeHeader string) (caches.Reader, []rangeutils.Range) {
// 尝试读取Partial cache
if len(rangeHeader) == 0 {
return nil, nil
}
ranges, ok := httpRequestParseRangeHeader(rangeHeader)
if !ok {
return nil, nil
}
pReader, pErr := storage.OpenReader(key+caches.SuffixPartial, useStale, true)
if pErr != nil {
return nil, nil
}
partialReader, ok := pReader.(*caches.PartialFileReader)
if !ok {
_ = pReader.Close()
return nil, nil
}
var isOk = false
defer func() {
if !isOk {
_ = pReader.Close()
}
}()
// 检查范围
for index, r := range ranges {
r1, ok := r.Convert(partialReader.MaxLength())
if !ok {
return nil, nil
}
r2, ok := partialReader.ContainsRange(r1)
if !ok {
return nil, nil
}
ranges[index] = r2
}
isOk = true
return pReader, ranges
}

View File

@@ -21,22 +21,22 @@ func (this *HTTPRequest) doReverseProxy() {
}
// 对URL的处理
stripPrefix := this.reverseProxy.StripPrefix
requestURI := this.reverseProxy.RequestURI
requestURIHasVariables := this.reverseProxy.RequestURIHasVariables()
var stripPrefix = this.reverseProxy.StripPrefix
var requestURI = this.reverseProxy.RequestURI
var requestURIHasVariables = this.reverseProxy.RequestURIHasVariables()
var requestHost = ""
if this.reverseProxy.RequestHostType == serverconfigs.RequestHostTypeCustomized {
requestHost = this.reverseProxy.RequestHost
}
requestHostHasVariables := this.reverseProxy.RequestHostHasVariables()
var requestHostHasVariables = this.reverseProxy.RequestHostHasVariables()
// 源站
requestCall := shared.NewRequestCall()
var requestCall = shared.NewRequestCall()
requestCall.Request = this.RawReq
requestCall.Formatter = this.Format
requestCall.Domain = this.ReqHost
origin := this.reverseProxy.NextOrigin(requestCall)
var origin = this.reverseProxy.NextOrigin(requestCall)
requestCall.CallResponseCallbacks(this.writer)
if origin == nil {
err := errors.New(this.URL() + ": no available origin sites for reverse proxy")
@@ -89,9 +89,9 @@ func (this *HTTPRequest) doReverseProxy() {
}
// 处理RequestURI中的问号
questionMark := strings.LastIndex(this.uri, "?")
var questionMark = strings.LastIndex(this.uri, "?")
if questionMark > 0 {
path := this.uri[:questionMark]
var path = this.uri[:questionMark]
if strings.Contains(path, "?") {
this.uri = path + "&" + this.uri[questionMark+1:]
}
@@ -102,7 +102,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
// 获取源站地址
originAddr := origin.Addr.PickAddress()
var originAddr = origin.Addr.PickAddress()
if origin.Addr.HostHasVariables() {
originAddr = this.Format(originAddr)
}
@@ -113,7 +113,7 @@ func (this *HTTPRequest) doReverseProxy() {
if requestHostHasVariables {
this.RawReq.Host = this.Format(requestHost)
} else {
this.RawReq.Host = this.reverseProxy.RequestHost
this.RawReq.Host = requestHost
}
this.RawReq.URL.Host = this.RawReq.Host
} else if this.reverseProxy.RequestHostType == serverconfigs.RequestHostTypeOrigin {
@@ -132,7 +132,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
// 重组请求URL
questionMark := strings.Index(this.uri, "?")
var questionMark = strings.Index(this.uri, "?")
if questionMark > -1 {
this.RawReq.URL.Path = this.uri[:questionMark]
this.RawReq.URL.RawQuery = this.uri[questionMark+1:]
@@ -159,7 +159,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
// 获取请求客户端
client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol)
client, err := SharedHTTPClientPool.Client(this, origin, originAddr, this.reverseProxy.ProxyProtocol, this.reverseProxy.FollowRedirects)
if err != nil {
remotelogs.Error("HTTP_REQUEST_REVERSE_PROXY", err.Error())
this.write50x(err, http.StatusBadGateway, true)
@@ -199,7 +199,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
} else {
// 是否为客户端方面的错误
isClientError := false
var isClientError = false
if ok {
if httpErr.Err == context.Canceled {
// 如果是服务器端主动关闭,则无需提示
@@ -254,7 +254,7 @@ func (this *HTTPRequest) doReverseProxy() {
if this.web.Charset != nil && this.web.Charset.IsOn && len(this.web.Charset.Charset) > 0 {
contentTypes, ok := resp.Header["Content-Type"]
if ok && len(contentTypes) > 0 {
contentType := contentTypes[0]
var contentType = contentTypes[0]
if _, found := textMimeMap[contentType]; found {
resp.Header["Content-Type"][0] = contentType + "; charset=" + this.web.Charset.Charset
}
@@ -284,8 +284,8 @@ func (this *HTTPRequest) doReverseProxy() {
}
// 输出到客户端
pool := this.bytePool(resp.ContentLength)
buf := pool.Get()
var pool = this.bytePool(resp.ContentLength)
var buf = pool.Get()
if shouldAutoFlush {
for {
n, readErr := resp.Body.Read(buf)
@@ -306,7 +306,7 @@ func (this *HTTPRequest) doReverseProxy() {
}
pool.Put(buf)
closeErr := resp.Body.Close()
var closeErr = resp.Body.Close()
if closeErr != nil {
if !this.canIgnore(closeErr) {
remotelogs.Warn("HTTP_REQUEST_REVERSE_PROXY", closeErr.Error())

View File

@@ -2,10 +2,12 @@ package nodes
import (
"fmt"
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/TeaOSLab/EdgeNode/internal/zero"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/types"
"io"
"io/fs"
"mime"
@@ -186,7 +188,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
// length
fileSize := stat.Size()
var fileSize = stat.Size()
// 支持 Last-Modified
modifiedTime := stat.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT")
@@ -231,6 +233,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
for _, v := range ifRangeHeaders {
if v == eTag || v == modifiedTime {
supportRange = true
break
}
}
if !supportRange {
@@ -239,7 +242,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
// 支持Range
rangeSet := [][]int64{}
var ranges = []rangeutils.Range{}
if supportRange {
contentRange := this.RawReq.Header.Get("Range")
if len(contentRange) > 0 {
@@ -249,36 +252,22 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return true
}
set, ok := httpRequestParseContentRange(contentRange)
set, ok := httpRequestParseRangeHeader(contentRange)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
if len(set) > 0 {
rangeSet = set
for _, arr := range rangeSet {
if arr[0] == -1 {
arr[0] = fileSize + arr[1]
arr[1] = fileSize - 1
if arr[0] < 0 {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
}
if arr[1] > 0 {
arr[1] = fileSize - 1
}
if arr[1] < 0 {
arr[1] = fileSize - 1
}
if arr[0] > arr[1] {
ranges = set
for k, r := range ranges {
r2, ok := r.Convert(fileSize)
if !ok {
this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
ranges[k] = r2
}
}
} else {
@@ -298,7 +287,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.processResponseHeaders(http.StatusOK)
// 在Range请求中不能缓存
if len(rangeSet) > 0 {
if len(ranges) > 0 {
this.cacheRef = nil // 不支持缓存
}
@@ -311,11 +300,11 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
pool.Put(buf)
}()
if len(rangeSet) == 1 {
respHeader.Set("Content-Range", "bytes "+strconv.FormatInt(rangeSet[0][0], 10)+"-"+strconv.FormatInt(rangeSet[0][1], 10)+"/"+strconv.FormatInt(fileSize, 10))
if len(ranges) == 1 {
respHeader.Set("Content-Range", ranges[0].ComposeContentRangeHeader(types.String(fileSize)))
this.writer.WriteHeader(http.StatusPartialContent)
ok, err := httpRequestReadRange(reader, buf, rangeSet[0][0], rangeSet[0][1], func(buf []byte, n int) error {
ok, err := httpRequestReadRange(reader, buf, ranges[0].Start(), ranges[0].End(), func(buf []byte, n int) error {
_, err := this.writer.Write(buf[:n])
return err
})
@@ -328,13 +317,13 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return true
}
} else if len(rangeSet) > 1 {
} else if len(ranges) > 1 {
boundary := httpRequestGenBoundary()
respHeader.Set("Content-Type", "multipart/byteranges; boundary="+boundary)
this.writer.WriteHeader(http.StatusPartialContent)
for index, set := range rangeSet {
for index, r := range ranges {
if index == 0 {
_, err = this.writer.WriteString("--" + boundary + "\r\n")
} else {
@@ -345,7 +334,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
return true
}
_, err = this.writer.WriteString("Content-Range: " + "bytes " + strconv.FormatInt(set[0], 10) + "-" + strconv.FormatInt(set[1], 10) + "/" + strconv.FormatInt(fileSize, 10) + "\r\n")
_, err = this.writer.WriteString("Content-Range: " + r.ComposeContentRangeHeader(types.String(fileSize)) + "\r\n")
if err != nil {
logs.Error(err)
return true
@@ -359,7 +348,7 @@ func (this *HTTPRequest) doRoot() (isBreak bool) {
}
}
ok, err := httpRequestReadRange(reader, buf, set[0], set[1], func(buf []byte, n int) error {
ok, err := httpRequestReadRange(reader, buf, r.Start(), r.End(), func(buf []byte, n int) error {
_, err := this.writer.Write(buf[:n])
return err
})

View File

@@ -11,7 +11,7 @@ func TestHTTPRequest_RedirectToHTTPS(t *testing.T) {
a := assert.NewAssertion(t)
{
req := &HTTPRequest{
Server: &serverconfigs.ServerConfig{
ReqServer: &serverconfigs.ServerConfig{
Web: &serverconfigs.HTTPWebConfig{
RedirectToHttps: &serverconfigs.HTTPRedirectToHTTPSConfig{},
},
@@ -22,7 +22,7 @@ func TestHTTPRequest_RedirectToHTTPS(t *testing.T) {
}
{
req := &HTTPRequest{
Server: &serverconfigs.ServerConfig{
ReqServer: &serverconfigs.ServerConfig{
Web: &serverconfigs.HTTPWebConfig{
RedirectToHttps: &serverconfigs.HTTPRedirectToHTTPSConfig{
IsOn: true,

View File

@@ -5,15 +5,21 @@ import (
"fmt"
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
"github.com/iwind/TeaGo/types"
"io"
"net/http"
"regexp"
"strconv"
"strings"
"sync/atomic"
)
// 其中的每个括号里的内容都在被引用,不能轻易修改
var contentRangeRegexp = regexp.MustCompile(`^bytes (\d+)-(\d+)/(\d+|\*)`)
// 分解Range
func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool) {
func httpRequestParseRangeHeader(rangeValue string) (result []rangeutils.Range, ok bool) {
// 参考RFChttps://tools.ietf.org/html/rfc7233
index := strings.Index(rangeValue, "=")
if index == -1 {
@@ -24,15 +30,15 @@ func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool)
return
}
rangeSetString := rangeValue[index+1:]
var rangeSetString = rangeValue[index+1:]
if len(rangeSetString) == 0 {
ok = true
return
}
pieces := strings.Split(rangeSetString, ", ")
var pieces = strings.Split(rangeSetString, ", ")
for _, piece := range pieces {
index := strings.Index(piece, "-")
index = strings.Index(piece, "-")
if index == -1 {
return
}
@@ -70,7 +76,7 @@ func httpRequestParseContentRange(rangeValue string) (result [][]int64, ok bool)
lastInt = -lastInt
}
result = append(result, []int64{firstInt, lastInt})
result = append(result, [2]int64{firstInt, lastInt})
}
ok = true
@@ -119,10 +125,25 @@ func httpRequestReadRange(reader io.Reader, buf []byte, start int64, end int64,
}
}
// 分解Content-Range
func httpRequestParseContentRangeHeader(contentRange string) (start int64, total int64) {
var matches = contentRangeRegexp.FindStringSubmatch(contentRange)
if len(matches) < 4 {
return -1, -1
}
start = types.Int64(matches[1])
var sizeString = matches[3]
if sizeString != "*" {
total = types.Int64(sizeString)
}
return
}
// 生成boundary
// 仿照Golang自带的函数multipart包
func httpRequestGenBoundary() string {
var buf [30]byte
var buf [8]byte
_, err := io.ReadFull(rand.Reader, buf[:])
if err != nil {
panic(err)
@@ -130,6 +151,21 @@ func httpRequestGenBoundary() string {
return fmt.Sprintf("%x", buf[:])
}
// 从content-type中读取boundary
func httpRequestParseBoundary(contentType string) string {
var delim = "boundary="
var boundaryIndex = strings.Index(contentType, delim)
if boundaryIndex < 0 {
return ""
}
var boundary = contentType[boundaryIndex+len(delim):]
semicolonIndex := strings.Index(boundary, ";")
if semicolonIndex >= 0 {
return boundary[:semicolonIndex]
}
return boundary
}
// 判断状态是否为跳转
func httpStatusIsRedirect(statusCode int) bool {
return statusCode == http.StatusPermanentRedirect ||
@@ -156,6 +192,9 @@ func httpRequestNextId() string {
// 检查是否可以接受某个编码
func httpAcceptEncoding(acceptEncodings string, encoding string) bool {
if len(acceptEncodings) == 0 {
return false
}
var pieces = strings.Split(acceptEncodings, ",")
for _, piece := range pieces {
var qualityIndex = strings.Index(piece, ";")
@@ -169,3 +208,20 @@ func httpAcceptEncoding(acceptEncodings string, encoding string) bool {
}
return false
}
// 分隔编码
func httpAcceptEncodings(acceptEncodings string) (encodings []string) {
if len(acceptEncodings) == 0 {
return
}
var pieces = strings.Split(acceptEncodings, ",")
for _, piece := range pieces {
var qualityIndex = strings.Index(piece, ";")
if qualityIndex >= 0 {
piece = piece[:qualityIndex]
}
encodings = append(encodings, strings.TrimSpace(piece))
}
return
}

View File

@@ -10,52 +10,91 @@ import (
"time"
)
func TestHTTPRequest_httpRequestParseContentRange(t *testing.T) {
a := assert.NewAssertion(t)
func TestHTTPRequest_httpRequestGenBoundary(t *testing.T) {
for i := 0; i < 10; i++ {
var boundary = httpRequestGenBoundary()
t.Log(boundary, "[", len(boundary), "bytes", "]")
}
}
func TestHTTPRequest_httpRequestParseBoundary(t *testing.T) {
var a = assert.NewAssertion(t)
a.IsTrue(httpRequestParseBoundary("multipart/byteranges") == "")
a.IsTrue(httpRequestParseBoundary("multipart/byteranges; boundary=123") == "123")
a.IsTrue(httpRequestParseBoundary("multipart/byteranges; boundary=123; 456") == "123")
}
func TestHTTPRequest_httpRequestParseRangeHeader(t *testing.T) {
var a = assert.NewAssertion(t)
{
_, ok := httpRequestParseContentRange("")
_, ok := httpRequestParseRangeHeader("")
a.IsFalse(ok)
}
{
_, ok := httpRequestParseContentRange("byte=")
_, ok := httpRequestParseRangeHeader("byte=")
a.IsFalse(ok)
}
{
_, ok := httpRequestParseContentRange("byte=")
_, ok := httpRequestParseRangeHeader("byte=")
a.IsFalse(ok)
}
{
set, ok := httpRequestParseContentRange("bytes=")
set, ok := httpRequestParseRangeHeader("bytes=")
a.IsTrue(ok)
a.IsTrue(len(set) == 0)
}
{
_, ok := httpRequestParseContentRange("bytes=60-50")
_, ok := httpRequestParseRangeHeader("bytes=60-50")
a.IsFalse(ok)
}
{
set, ok := httpRequestParseContentRange("bytes=0-50")
set, ok := httpRequestParseRangeHeader("bytes=0-50")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
}
{
set, ok := httpRequestParseContentRange("bytes=0-")
set, ok := httpRequestParseRangeHeader("bytes=0-")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
if len(set) > 0 {
a.IsTrue(set[0][0] == 0)
}
t.Log(set)
}
{
set, ok := httpRequestParseRangeHeader("bytes=-50")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
}
{
set, ok := httpRequestParseContentRange("bytes=-50")
set, ok := httpRequestParseRangeHeader("bytes=0-50, 60-100")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
}
}
func TestHTTPRequest_httpRequestParseContentRangeHeader(t *testing.T) {
{
set, ok := httpRequestParseContentRange("bytes=0-50, 60-100")
a.IsTrue(ok)
a.IsTrue(len(set) > 0)
t.Log(set)
var c1 = "bytes 0-100/*"
t.Log(httpRequestParseContentRangeHeader(c1))
}
{
var c1 = "bytes 30-100/*"
t.Log(httpRequestParseContentRangeHeader(c1))
}
{
var c1 = "bytes1 0-100/*"
t.Log(httpRequestParseContentRangeHeader(c1))
}
}
func BenchmarkHTTPRequest_httpRequestParseContentRangeHeader(b *testing.B) {
for i := 0; i < b.N; i++ {
var c1 = "bytes 0-100/*"
httpRequestParseContentRangeHeader(c1)
}
}

View File

@@ -35,11 +35,15 @@ func (this *HTTPRequest) doWAFRequest() (blocked bool) {
}
// 是否在全局名单中
if !iplibrary.AllowIP(remoteAddr, this.ReqServer.Id) {
canGoNext, isInAllowedList := iplibrary.AllowIP(remoteAddr, this.ReqServer.Id)
if !canGoNext {
this.disableLog = true
this.Close()
return true
}
if isInAllowedList {
return false
}
// 检查是否在临时黑名单中
if waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeService, this.ReqServer.Id, remoteAddr) || waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, remoteAddr) {

View File

@@ -28,15 +28,13 @@ import (
"io/ioutil"
"net"
"net/http"
"net/textproto"
"os"
"path/filepath"
"strings"
"sync/atomic"
)
// webp相关配置
const webpSuffix = "@GOEDGE_WEBP"
var webpMaxBufferSize int64 = 1_000_000_000
var webpTotalBufferSize int64 = 0
@@ -66,17 +64,25 @@ type HTTPWriter struct {
isOk bool // 是否完全成功
isFinished bool // 是否已完成
// Partial
isPartial bool
partialFileIsNew bool
// WebP
webpIsEncoding bool
webpOriginContentType string
// Compression
compressionConfig *serverconfigs.HTTPCompressionConfig
compressionConfig *serverconfigs.HTTPCompressionConfig
compressionCacheWriter caches.Writer
// Cache
cacheStorage caches.StorageInterface
cacheWriter caches.Writer
cacheIsFinished bool
cacheReader caches.Reader
cacheReaderSuffix string
}
// NewHTTPWriter 包装对象
@@ -95,13 +101,28 @@ func (this *HTTPWriter) Prepare(resp *http.Response, size int64, status int, ena
this.size = size
this.statusCode = status
if resp != nil {
// 是否为区间请求
this.isPartial = status == http.StatusPartialContent
// 不支持对GET以外的方法返回的Partial内容的缓存
if this.isPartial && this.req.Method() != http.MethodGet {
enableCache = false
}
if resp != nil && resp.Body != nil {
cacheReader, ok := resp.Body.(caches.Reader)
if ok {
this.cacheReader = cacheReader
}
this.rawReader = resp.Body
if enableCache {
this.PrepareCache(resp, size)
}
this.PrepareWebP(resp, size)
if !this.isPartial {
this.PrepareWebP(resp, size)
}
this.PrepareCompression(resp, size)
}
@@ -135,12 +156,21 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
var addStatusHeader = this.req.web != nil && this.req.web.Cache != nil && this.req.web.Cache.AddStatusHeader
// 不支持Range
if len(this.Header().Get("Content-Range")) > 0 {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, not supported Content-Range")
if this.isPartial {
if !cacheRef.AllowPartialContent {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, not supported partial content")
}
return
}
if this.cacheStorage.Policy().Type != serverconfigs.CachePolicyStorageFile {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, not supported partial content in memory storage")
}
return
}
return
}
// 如果允许 ChunkedEncoding就无需尺寸的判断因为此时的 size 为 -1
@@ -151,8 +181,18 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
return
}
if size >= 0 && ((cacheRef.MaxSizeBytes() > 0 && size > cacheRef.MaxSizeBytes()) ||
(cachePolicy.MaxSizeBytes() > 0 && size > cachePolicy.MaxSizeBytes()) || (cacheRef.MinSizeBytes() > size)) {
var contentSize = size
if this.isPartial {
// 从Content-Range中读取内容总长度
var contentRange = this.Header().Get("Content-Range")
_, totalSize := httpRequestParseContentRangeHeader(contentRange)
if totalSize > 0 {
contentSize = totalSize
}
}
if contentSize >= 0 && ((cacheRef.MaxSizeBytes() > 0 && contentSize > cacheRef.MaxSizeBytes()) ||
(cachePolicy.MaxSizeBytes() > 0 && contentSize > cachePolicy.MaxSizeBytes()) || (cacheRef.MinSizeBytes() > contentSize)) {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, Content-Length")
@@ -161,7 +201,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
// 检查状态
if len(cacheRef.Status) > 0 && !lists.ContainsInt(cacheRef.Status, this.StatusCode()) {
if !cacheRef.MatchStatus(this.StatusCode()) {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, Status: "+types.String(this.StatusCode()))
@@ -171,7 +211,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
// Cache-Control
if len(cacheRef.SkipResponseCacheControlValues) > 0 {
var cacheControl = this.Header().Get("Cache-Control")
var cacheControl = this.GetHeader("Cache-Control")
if len(cacheControl) > 0 {
values := strings.Split(cacheControl, ",")
for _, value := range values {
@@ -187,7 +227,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
// Set-Cookie
if cacheRef.SkipResponseSetCookie && len(this.Header().Get("Set-Cookie")) > 0 {
if cacheRef.SkipResponseSetCookie && len(this.GetHeader("Set-Cookie")) > 0 {
this.req.varMapping["cache.status"] = "BYPASS"
if addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, Set-Cookie")
@@ -220,7 +260,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
this.cacheStorage = storage
life := cacheRef.LifeSeconds()
var life = cacheRef.LifeSeconds()
if life <= 0 {
life = 60
@@ -228,7 +268,7 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
// 支持源站设置的max-age
if this.req.web.Cache != nil && this.req.web.Cache.EnableCacheControlMaxAge {
var cacheControl = this.Header().Get("Cache-Control")
var cacheControl = this.GetHeader("Cache-Control")
var pieces = strings.Split(cacheControl, ";")
for _, piece := range pieces {
var eqIndex = strings.Index(piece, "=")
@@ -243,8 +283,15 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
var expiredAt = utils.UnixTime() + life
var cacheKey = this.req.cacheKey
cacheWriter, err := storage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), size, false)
if this.isPartial {
cacheKey += caches.SuffixPartial
}
cacheWriter, err := storage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), size, cacheRef.MaxSizeBytes(), this.isPartial)
if err != nil {
if err == caches.ErrEntityTooLarge && addStatusHeader {
this.Header().Set("X-Cache", "BYPASS, entity too large")
}
if !caches.CanIgnoreErr(err) {
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
}
@@ -252,9 +299,16 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
this.cacheWriter = cacheWriter
if this.isPartial {
this.partialFileIsNew = cacheWriter.(*caches.PartialFileWriter).IsNew()
}
// 写入Header
for k, v := range this.Header() {
for _, v1 := range v {
if this.isPartial && k == "Content-Type" && strings.Contains(v1, "multipart/byteranges") {
continue
}
_, err = cacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
remotelogs.Error("HTTP_WRITER", "write cache failed: "+err.Error())
@@ -265,6 +319,110 @@ func (this *HTTPWriter) PrepareCache(resp *http.Response, size int64) {
}
}
if this.isPartial {
// content-range
var contentRange = this.GetHeader("Content-Range")
if len(contentRange) > 0 {
start, total := httpRequestParseContentRangeHeader(contentRange)
if start < 0 {
return
}
if total > 0 {
partialWriter, ok := cacheWriter.(*caches.PartialFileWriter)
if !ok {
return
}
partialWriter.SetBodyLength(total)
}
var filterReader = readers.NewFilterReaderCloser(resp.Body)
this.cacheIsFinished = true
var hasError = false
filterReader.Add(func(p []byte, err error) error {
if hasError {
return nil
}
var l = len(p)
if l == 0 {
return nil
}
defer func() {
start += int64(l)
}()
err = cacheWriter.WriteAt(start, p)
if err != nil {
this.cacheIsFinished = false
hasError = true
}
return nil
})
resp.Body = filterReader
this.rawReader = filterReader
return
}
// multipart/byteranges
var contentType = this.GetHeader("Content-Type")
if strings.Contains(contentType, "multipart/byteranges") {
partialWriter, ok := cacheWriter.(*caches.PartialFileWriter)
if !ok {
return
}
var boundary = httpRequestParseBoundary(contentType)
if len(boundary) == 0 {
return
}
var reader = readers.NewByteRangesReaderCloser(resp.Body, boundary)
var contentTypeWritten = false
this.cacheIsFinished = true
var hasError = false
var writtenTotal = false
reader.OnPartRead(func(start int64, end int64, total int64, data []byte, header textproto.MIMEHeader) {
// TODO 如果 total 超出缓存限制则不写入缓存数据并且记录到某个内存表中下次不再OpenWriter
if hasError {
return
}
// 写入total
if !writtenTotal && total > 0 {
partialWriter.SetBodyLength(total)
writtenTotal = true
}
// 写入Content-Type
if partialWriter.IsNew() && !contentTypeWritten {
var realContentType = header.Get("Content-Type")
if len(realContentType) > 0 {
var h = []byte("Content-Type:" + realContentType + "\n")
err = partialWriter.AppendHeader(h)
if err != nil {
hasError = true
this.cacheIsFinished = false
return
}
}
contentTypeWritten = true
}
err := cacheWriter.WriteAt(start, data)
if err != nil {
hasError = true
this.cacheIsFinished = false
}
})
resp.Body = reader
this.rawReader = reader
}
return
}
var cacheReader = readers.NewTeeReaderCloser(resp.Body, this.cacheWriter)
resp.Body = cacheReader
this.rawReader = cacheReader
@@ -284,7 +442,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
return
}
var contentType = this.Header().Get("Content-Type")
var contentType = this.GetHeader("Content-Type")
if this.req.web != nil &&
this.req.web.WebP != nil &&
@@ -302,7 +460,7 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
return
}
var contentEncoding = resp.Header.Get("Content-Encoding")
var contentEncoding = this.GetHeader("Content-Encoding")
switch contentEncoding {
case "gzip", "deflate", "br":
reader, err := compressions.NewReader(resp.Body, contentEncoding)
@@ -329,8 +487,17 @@ func (this *HTTPWriter) PrepareWebP(resp *http.Response, size int64) {
// PrepareCompression 准备压缩
func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
var method = this.req.Method()
if method == http.MethodHead {
return
}
if this.StatusCode() == http.StatusNoContent {
return
}
var acceptEncodings = this.req.RawReq.Header.Get("Accept-Encoding")
var contentEncoding = this.Header().Get("Content-Encoding")
var contentEncoding = this.GetHeader("Content-Encoding")
if this.compressionConfig == nil || !this.compressionConfig.IsOn {
if lists.ContainsString([]string{"gzip", "deflate", "br"}, contentEncoding) && !httpAcceptEncoding(acceptEncodings, contentEncoding) {
@@ -355,11 +522,7 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
}
// 尺寸和类型
var contentType = this.Header().Get("Content-Type")
if len(contentType) == 0 {
// 如果没有显式设置Content-Type我们就认为是 text/html
contentType = "text/html"
}
var contentType = this.GetHeader("Content-Type")
if !this.compressionConfig.MatchResponse(contentType, size, filepath.Ext(this.req.Path()), this.req.Format) {
return
}
@@ -389,19 +552,76 @@ func (this *HTTPWriter) PrepareCompression(resp *http.Response, size int64) {
resp.Body = reader
}
// 需要放在compression cache writer之前
var header = this.rawWriter.Header()
header.Set("Content-Encoding", compressionEncoding)
header.Set("Vary", "Accept-Encoding")
header.Del("Content-Length")
// compression cache writer
// 只有在本身内容已经缓存的情况下才会写入缓存防止同时写入缓存导致IO负载升高
var cacheRef = this.req.cacheRef
if !this.isPartial &&
this.cacheStorage != nil &&
cacheRef != nil &&
(this.cacheReader != nil || (this.cacheStorage.Policy().SyncCompressionCache && this.cacheWriter != nil)) &&
!this.webpIsEncoding {
var cacheKey = ""
var expiredAt int64 = 0
if this.cacheReader != nil {
cacheKey = this.req.cacheKey
expiredAt = this.cacheReader.ExpiresAt()
} else if this.cacheWriter != nil {
cacheKey = this.cacheWriter.Key()
expiredAt = this.cacheWriter.ExpiredAt()
}
if len(this.cacheReaderSuffix) > 0 {
cacheKey += this.cacheReaderSuffix
}
compressionCacheWriter, err := this.cacheStorage.OpenWriter(cacheKey+caches.SuffixCompression+compressionEncoding, expiredAt, this.StatusCode(), -1, cacheRef.MaxSizeBytes(), false)
if err != nil {
return
}
// 写入Header
for k, v := range this.Header() {
for _, v1 := range v {
_, err = compressionCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
remotelogs.Error("HTTP_WRITER", "write compression cache failed: "+err.Error())
_ = compressionCacheWriter.Discard()
compressionCacheWriter = nil
return
}
}
}
if compressionCacheWriter != nil {
this.compressionCacheWriter = compressionCacheWriter
var teeWriter = writers.NewTeeWriterCloser(this.writer, compressionCacheWriter)
teeWriter.OnFail(func(err error) {
_ = compressionCacheWriter.Discard()
this.compressionCacheWriter = nil
})
this.writer = teeWriter
}
}
// compression writer
var err error = nil
compressionWriter, err := compressions.NewWriter(this.writer, compressionType, int(this.compressionConfig.Level))
if err != nil {
remotelogs.Error("HTTP_WRITER", err.Error())
header.Del("Content-Encoding")
if this.compressionCacheWriter != nil {
_ = this.compressionCacheWriter.Discard()
}
return
}
this.writer = compressionWriter
header := this.rawWriter.Header()
header.Set("Content-Encoding", compressionEncoding)
header.Set("Vary", "Accept-Encoding")
header.Del("Content-Length")
}
// SetCompression 设置内容压缩配置
@@ -422,6 +642,11 @@ func (this *HTTPWriter) Header() http.Header {
return this.rawWriter.Header()
}
// GetHeader 读取Header值
func (this *HTTPWriter) GetHeader(name string) string {
return this.Header().Get(name)
}
// DeleteHeader 删除Header
func (this *HTTPWriter) DeleteHeader(name string) {
this.rawWriter.Header().Del(name)
@@ -557,13 +782,26 @@ func (this *HTTPWriter) Close() {
var webpCacheWriter caches.Writer
// 准备WebP Cache
if this.cacheWriter != nil {
var cacheKey = this.cacheWriter.Key() + webpSuffix
if this.cacheReader != nil || this.cacheWriter != nil {
var cacheKey = ""
var expiredAt int64 = 0
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, this.cacheWriter.ExpiredAt(), this.StatusCode(), -1, false)
if this.cacheReader != nil {
cacheKey = this.req.cacheKey + caches.SuffixWebP
expiredAt = this.cacheReader.ExpiresAt()
} else if this.cacheWriter != nil {
cacheKey = this.cacheWriter.Key() + caches.SuffixWebP
expiredAt = this.cacheWriter.ExpiredAt()
}
webpCacheWriter, _ = this.cacheStorage.OpenWriter(cacheKey, expiredAt, this.StatusCode(), -1, -1, false)
if webpCacheWriter != nil {
// 写入Header
for k, v := range this.Header() {
// 这里是原始的数据,不需要内容编码
if k == "Content-Encoding" || k == "Transfer-Encoding" {
continue
}
for _, v1 := range v {
_, err := webpCacheWriter.WriteHeader([]byte(k + ":" + v1 + "\n"))
if err != nil {
@@ -680,19 +918,45 @@ func (this *HTTPWriter) Close() {
// 缓存
if this.cacheWriter != nil {
if this.isOk && this.cacheIsFinished {
// 对比Content-Length
var contentLengthString = this.Header().Get("Content-Length")
if len(contentLengthString) > 0 {
var contentLength = types.Int64(contentLengthString)
if contentLength != this.cacheWriter.BodySize() {
this.isOk = false
_ = this.cacheWriter.Discard()
// 对比缓存前后的Content-Length
var method = this.req.Method()
if method != http.MethodHead && this.StatusCode() != http.StatusNoContent && !this.isPartial {
var contentLengthString = this.GetHeader("Content-Length")
if len(contentLengthString) > 0 {
var contentLength = types.Int64(contentLengthString)
if contentLength != this.cacheWriter.BodySize() {
this.isOk = false
_ = this.cacheWriter.Discard()
this.cacheWriter = nil
}
}
}
if this.isOk {
if this.isOk && this.cacheWriter != nil {
err := this.cacheWriter.Close()
if err == nil {
if !this.isPartial || this.partialFileIsNew {
var expiredAt = this.cacheWriter.ExpiredAt()
this.cacheStorage.AddToList(&caches.Item{
Type: this.cacheWriter.ItemType(),
Key: this.cacheWriter.Key(),
ExpiredAt: expiredAt,
StaleAt: expiredAt + int64(this.calculateStaleLife()),
HeaderSize: this.cacheWriter.HeaderSize(),
BodySize: this.cacheWriter.BodySize(),
Host: this.req.ReqHost,
ServerId: this.req.ReqServer.Id,
})
}
}
}
} else {
if !this.isPartial || !this.cacheIsFinished {
_ = this.cacheWriter.Discard()
} else {
// Partial的文件内容不删除
err := this.cacheWriter.Close()
if err == nil && this.partialFileIsNew {
var expiredAt = this.cacheWriter.ExpiredAt()
this.cacheStorage.AddToList(&caches.Item{
Type: this.cacheWriter.ItemType(),
@@ -706,8 +970,27 @@ func (this *HTTPWriter) Close() {
})
}
}
}
}
if this.compressionCacheWriter != nil {
if this.isOk {
err := this.compressionCacheWriter.Close()
if err == nil {
var expiredAt = this.compressionCacheWriter.ExpiredAt()
this.cacheStorage.AddToList(&caches.Item{
Type: this.compressionCacheWriter.ItemType(),
Key: this.compressionCacheWriter.Key(),
ExpiredAt: expiredAt,
StaleAt: expiredAt + int64(this.calculateStaleLife()),
HeaderSize: this.compressionCacheWriter.HeaderSize(),
BodySize: this.compressionCacheWriter.BodySize(),
Host: this.req.ReqHost,
ServerId: this.req.ReqServer.Id,
})
}
} else {
_ = this.cacheWriter.Discard()
_ = this.compressionCacheWriter.Discard()
}
}
@@ -744,7 +1027,7 @@ func (this *HTTPWriter) calculateStaleLife() int {
// 从Header中读取stale-if-error
var isDefinedInHeader = false
if staleConfig.SupportStaleIfErrorHeader {
var cacheControl = this.Header().Get("Cache-Control")
var cacheControl = this.GetHeader("Cache-Control")
var pieces = strings.Split(cacheControl, ",")
for _, piece := range pieces {
var eqIndex = strings.Index(piece, "=")

View File

@@ -34,22 +34,27 @@ func (this *BaseListener) CountActiveConnections() int {
func (this *BaseListener) buildTLSConfig() *tls.Config {
return &tls.Config{
Certificates: nil,
GetConfigForClient: func(configInfo *tls.ClientHelloInfo) (config *tls.Config, e error) {
ssl, _, err := this.matchSSL(configInfo.ServerName)
GetConfigForClient: func(clientInfo *tls.ClientHelloInfo) (config *tls.Config, e error) {
tlsPolicy, _, err := this.matchSSL(clientInfo.ServerName)
if err != nil {
return nil, err
}
return ssl.TLSConfig(), nil
tlsPolicy.CheckOCSP()
return tlsPolicy.TLSConfig(), nil
},
GetCertificate: func(certInfo *tls.ClientHelloInfo) (certificate *tls.Certificate, e error) {
_, cert, err := this.matchSSL(certInfo.ServerName)
GetCertificate: func(clientInfo *tls.ClientHelloInfo) (certificate *tls.Certificate, e error) {
tlsPolicy, cert, err := this.matchSSL(clientInfo.ServerName)
if err != nil {
return nil, err
}
if cert == nil {
return nil, errors.New("no ssl certs found for '" + certInfo.ServerName + "'")
return nil, errors.New("no ssl certs found for '" + clientInfo.ServerName + "'")
}
tlsPolicy.CheckOCSP()
return cert, nil
},
}

View File

@@ -22,21 +22,23 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/trackers"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/andybalholm/brotli"
"github.com/go-yaml/yaml"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/logs"
"github.com/iwind/TeaGo/maps"
"github.com/iwind/TeaGo/types"
"github.com/iwind/gosock/pkg/gosock"
"gopkg.in/yaml.v3"
"io/ioutil"
"log"
"os"
"os/exec"
"os/signal"
"runtime"
"runtime/debug"
"sort"
"sync"
"syscall"
"time"
)
@@ -96,6 +98,9 @@ func (this *Node) Start() {
// 处理异常
this.handlePanic()
// 监听signal
this.listenSignals()
// 启动事件
events.Notify(events.EventStart)
@@ -593,6 +598,19 @@ func (this *Node) checkClusterConfig() error {
return nil
}
// 监听一些信号
func (this *Node) listenSignals() {
var queue = make(chan os.Signal, 8)
signal.Notify(queue, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT)
goman.New(func() {
for range queue {
time.Sleep(100 * time.Millisecond)
utils.Exit()
return
}
})
}
// 监听本地sock
func (this *Node) listenSock() error {
// 检查是否在运行
@@ -631,7 +649,7 @@ func (this *Node) listenSock() error {
// 退出主进程
events.Notify(events.EventQuit)
os.Exit(0)
utils.Exit()
case "quit":
_ = cmd.ReplyOk()
_ = this.sock.Close()
@@ -643,7 +661,7 @@ func (this *Node) listenSock() error {
for {
countActiveConnections := sharedListenerManager.TotalActiveConnections()
if countActiveConnections <= 0 {
os.Exit(0)
utils.Exit()
return
}
time.Sleep(1 * time.Second)
@@ -742,6 +760,17 @@ func (this *Node) listenSock() error {
runtime.GC()
debug.FreeOSMemory()
_ = cmd.ReplyOk()
case "reload":
err := this.syncConfig(0)
if err != nil {
_ = cmd.Reply(&gosock.Command{
Params: map[string]interface{}{
"error": err.Error(),
},
})
} else {
_ = cmd.ReplyOk()
}
}
})
@@ -767,6 +796,7 @@ func (this *Node) onReload(config *nodeconfigs.NodeConfig) {
// 缓存策略
caches.SharedManager.MaxDiskCapacity = config.MaxCacheDiskCapacity
caches.SharedManager.MaxMemoryCapacity = config.MaxCacheMemoryCapacity
caches.SharedManager.DiskDir = config.CacheDiskDir
if len(config.HTTPCachePolicies) > 0 {
caches.SharedManager.UpdatePolicies(config.HTTPCachePolicies)
} else {

View File

@@ -14,8 +14,8 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/lists"
"github.com/iwind/TeaGo/maps"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/disk"
"github.com/shirou/gopsutil/v3/cpu"
"github.com/shirou/gopsutil/v3/disk"
"golang.org/x/sys/unix"
"os"
"runtime"

View File

@@ -1,7 +1,7 @@
package nodes
import (
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/v3/cpu"
"testing"
"time"
)

View File

@@ -7,8 +7,8 @@ import (
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
"github.com/TeaOSLab/EdgeNode/internal/monitor"
"github.com/iwind/TeaGo/maps"
"github.com/shirou/gopsutil/load"
"github.com/shirou/gopsutil/mem"
"github.com/shirou/gopsutil/v3/load"
"github.com/shirou/gopsutil/v3/mem"
)
// 更新内存

View File

@@ -4,8 +4,8 @@ package nodes
import (
"context"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/mem"
"github.com/shirou/gopsutil/v3/cpu"
"github.com/shirou/gopsutil/v3/mem"
"math"
"sync"
"time"

View File

@@ -0,0 +1,86 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes
import (
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
"github.com/TeaOSLab/EdgeNode/internal/events"
"github.com/TeaOSLab/EdgeNode/internal/goman"
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/rpc"
"github.com/iwind/TeaGo/Tea"
"time"
)
var sharedOCSPTask = NewOCSPUpdateTask()
func init() {
events.On(events.EventLoaded, func() {
sharedOCSPTask.version = sharedNodeConfig.OCSPVersion
goman.New(func() {
sharedOCSPTask.Start()
})
})
events.On(events.EventQuit, func() {
sharedOCSPTask.Stop()
})
}
// OCSPUpdateTask 更新OCSP任务
type OCSPUpdateTask struct {
version int64
ticker *time.Ticker
}
func NewOCSPUpdateTask() *OCSPUpdateTask {
var ticker = time.NewTicker(1 * time.Minute)
if Tea.IsTesting() {
ticker = time.NewTicker(10 * time.Second)
}
return &OCSPUpdateTask{
ticker: ticker,
}
}
func (this *OCSPUpdateTask) Start() {
for range this.ticker.C {
err := this.Loop()
if err != nil {
remotelogs.Warn("OCSPUpdateTask", "update ocsp failed: "+err.Error())
}
}
}
func (this *OCSPUpdateTask) Loop() error {
rpcClient, err := rpc.SharedRPC()
if err != nil {
return err
}
resp, err := rpcClient.SSLCertService().ListUpdatedSSLCertOCSP(rpcClient.Context(), &pb.ListUpdatedSSLCertOCSPRequest{
Version: this.version,
Size: 100,
})
if err != nil {
return err
}
for _, ocsp := range resp.SslCertOCSP {
// 更新OCSP
if sharedNodeConfig != nil {
sharedNodeConfig.UpdateCertOCSP(ocsp.SslCertId, ocsp.Data, ocsp.ExpiresAt)
}
// 修改版本
this.version = ocsp.Version
}
return nil
}
func (this *OCSPUpdateTask) Stop() {
this.ticker.Stop()
}

View File

@@ -0,0 +1,16 @@
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package nodes_test
import (
"github.com/TeaOSLab/EdgeNode/internal/nodes"
"testing"
)
func TestOCSPUpdateTask_Loop(t *testing.T) {
var task = &nodes.OCSPUpdateTask{}
err := task.Loop()
if err != nil {
t.Fatal(err)
}
}

View File

@@ -228,7 +228,7 @@ func (this *UpgradeManager) unzip(zipPath string) error {
func (this *UpgradeManager) restart() error {
// 重新启动
if DaemonIsOn && DaemonPid == os.Getppid() {
os.Exit(0) // TODO 试着更优雅重启
utils.Exit() // TODO 试着更优雅重启
} else {
exe, err := os.Executable()
if err != nil {
@@ -238,6 +238,9 @@ func (this *UpgradeManager) restart() error {
// quit
events.Notify(events.EventQuit)
// terminated
events.Notify(events.EventTerminated)
// 启动
cmd := exec.Command(exe, "start")
err = cmd.Start()

View File

@@ -4,13 +4,14 @@ package re
import (
"regexp"
"regexp/syntax"
"strings"
)
var prefixReg = regexp.MustCompile(`^\(\?([\w\s]+)\)`) // (?x)
var prefixReg2 = regexp.MustCompile(`^\(\?([\w\s]*:)`) // (?x: ...
var braceZero = regexp.MustCompile(`^{\s*0*\s*}`) // {0}
var braceZero2 = regexp.MustCompile(`^{\s*0*\s*,`) // {0, x}
var braceZeroReg = regexp.MustCompile(`^{\s*0*\s*}`) // {0}
var braceZeroReg2 = regexp.MustCompile(`^{\s*0*\s*,`) // {0, x}
type Regexp struct {
exp string
@@ -53,8 +54,6 @@ func (this *Regexp) init() {
return
}
//var keywords = []string{}
var exp = strings.TrimSpace(this.exp)
// 去掉前面的(?...)
@@ -68,9 +67,23 @@ func (this *Regexp) init() {
}
var keywords = this.ParseKeywords(exp)
this.keywords = keywords
if len(keywords) > 0 {
this.keywordsMap = NewRuneTree(keywords)
var filteredKeywords = []string{}
var minLength = 1
var isValid = true
for _, keyword := range keywords {
if len(keyword) <= minLength {
isValid = false
break
}
}
if isValid {
filteredKeywords = keywords
}
this.keywords = filteredKeywords
if len(filteredKeywords) > 0 {
this.keywordsMap = NewRuneTree(filteredKeywords)
}
}
@@ -96,6 +109,7 @@ func (this *Regexp) MatchString(s string) bool {
return true
}
}
return this.rawRegexp.MatchString(s)
}
@@ -113,104 +127,106 @@ func (this *Regexp) Match(s []byte) bool {
}
// ParseKeywords 提取表达式中的关键词
// TODO 支持嵌套,类似于 A(abc|bcd)
// TODO 支持 (?:xxx)
// TODO 支持 abc)(bcd)(efg)
func (this *Regexp) ParseKeywords(exp string) []string {
var keywords = []string{}
func (this *Regexp) ParseKeywords(exp string) (keywords []string) {
if len(exp) == 0 {
return nil
}
var runes = []rune(exp)
// (a|b|c)
reg, err := regexp.Compile(exp)
if err == nil {
var countSub = reg.NumSubexp()
if countSub == 1 {
beginIndex := this.indexOfSymbol(runes, '(')
if beginIndex >= 0 {
runes = runes[beginIndex+1:]
symbolIndex := this.indexOfSymbol(runes, ')')
if symbolIndex > 0 && this.isPlain(runes[symbolIndex+1:]) {
runes = runes[:symbolIndex]
if len(runes) == 0 {
return nil
}
}
}
}
reg, err := syntax.Parse(exp, syntax.Perl)
if err != nil {
return nil
}
var lastIndex = 0
for index, r := range runes {
if r == '|' {
if index > 0 && runes[index-1] != '\\' {
var ks = this.parseKeyword(runes[lastIndex:index])
if len(ks) > 0 {
keywords = append(keywords, string(ks))
if len(reg.Sub) == 0 {
var keywordRunes = this.parseKeyword(reg.String())
if len(keywordRunes) > 0 {
keywords = append(keywords, string(keywordRunes))
}
return
}
if len(reg.Sub) == 1 {
if reg.Op == syntax.OpStar || reg.Op == syntax.OpQuest || reg.Op == syntax.OpRepeat {
return nil
}
return this.ParseKeywords(reg.Sub[0].String())
}
switch reg.Op {
case syntax.OpConcat:
var prevKeywords = []string{}
var isStarted bool
for _, sub := range reg.Sub {
if sub.String() == `\b` {
if isStarted {
break
}
continue
}
if sub.Op != syntax.OpLiteral && sub.Op != syntax.OpCapture && sub.Op != syntax.OpAlternate {
if isStarted {
break
}
continue
}
var subKeywords = this.ParseKeywords(sub.String())
if len(subKeywords) > 0 {
if !isStarted {
prevKeywords = subKeywords
isStarted = true
} else {
return nil
for _, prevKeyword := range prevKeywords {
for _, subKeyword := range subKeywords {
keywords = append(keywords, prevKeyword+subKeyword)
}
}
prevKeywords = keywords
}
lastIndex = index + 1
} else {
break
}
}
}
if lastIndex == 0 {
var ks = this.parseKeyword(runes)
if len(ks) > 0 {
keywords = append(keywords, string(ks))
} else {
return nil
if len(prevKeywords) > 0 && len(keywords) == 0 {
keywords = prevKeywords
}
} else if lastIndex > 0 {
var ks = this.parseKeyword(runes[lastIndex:])
if len(ks) > 0 {
keywords = append(keywords, string(ks))
} else {
return nil
case syntax.OpAlternate:
for _, sub := range reg.Sub {
var subKeywords = this.ParseKeywords(sub.String())
if len(subKeywords) == 0 {
keywords = nil
return
}
keywords = append(keywords, subKeywords...)
}
}
return keywords
return
}
func (this *Regexp) parseKeyword(keyword []rune) (result []rune) {
if len(keyword) == 0 {
return
func (this *Regexp) parseKeyword(subExp string) (result []rune) {
if len(subExp) == 0 {
return nil
}
// remove first \b
for index, r := range keyword {
if r == '\b' {
keyword = keyword[index+1:]
break
} else if r != '\t' && r != '\r' && r != '\n' && r != ' ' {
break
// 去除开始和结尾的()
if subExp[0] == '(' && subExp[len(subExp)-1] == ')' {
subExp = subExp[1 : len(subExp)-1]
if len(subExp) == 0 {
return
}
}
if len(keyword) == 0 {
return
}
for index, r := range keyword {
if index == 0 && r == '^' {
continue
}
if r == '(' || r == ')' {
if index == 0 {
return nil
}
if keyword[index-1] != '\\' {
return nil
}
}
var runes = []rune(subExp)
for index, r := range runes {
if r == '[' || r == '{' || r == '.' || r == '+' || r == '$' {
if index == 0 {
return nil
return
}
if keyword[index-1] != '\\' {
if r == '{' && (braceZero.MatchString(string(keyword[index:])) || braceZero2.MatchString(string(keyword[index:]))) { // r {0, ...}
if runes[index-1] != '\\' {
if r == '{' && (braceZeroReg.MatchString(subExp[index:])) || braceZeroReg2.MatchString(subExp[index:]) { // r {0, ...}
if len(result) == 0 {
return nil
}
return result[:len(result)-1]
}
@@ -219,36 +235,40 @@ func (this *Regexp) parseKeyword(keyword []rune) (result []rune) {
}
if r == '?' || r == '*' {
if index == 0 {
return nil
return
}
if runes[index-1] != '\\' {
if len(result) > 0 {
return result[:len(result)-1]
}
return
}
return result[:len(result)-1]
}
if r == '\\' || r == '\b' {
// TODO 将来更精细的处理 \d, \s, \$等
break
}
if (r == 'n' || r == 't' || r == 'a' || r == 'f' || r == 'r' || r == 'v' || r == 'x') && index > 0 && runes[index-1] == '\\' {
switch r {
case 'n':
r = '\n'
case 't':
r = '\t'
case 'f':
r = '\f'
case 'r':
r = '\r'
case 'v':
r = '\v'
case 'a':
r = '\a'
case 'x':
return
}
}
if r == '\\' {
continue
}
result = append(result, r)
}
return
}
// 查找符号位置
func (this *Regexp) indexOfSymbol(runes []rune, symbol rune) int {
for index, c := range runes {
if c == symbol && (index == 0 || runes[index-1] != '\\') {
return index
}
}
return -1
}
// 是否可视为为普通字符
func (this *Regexp) isPlain(runes []rune) bool {
for _, r := range []rune{'|', '(', ')'} {
if this.indexOfSymbol(runes, r) >= 0 {
return false
}
}
return true
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/re"
"github.com/iwind/TeaGo/assert"
"regexp"
"strings"
"testing"
)
@@ -27,12 +28,14 @@ func TestRegexp_MatchString(t *testing.T) {
var r = re.MustCompile("abc")
a.IsTrue(r.MatchString("abc"))
a.IsFalse(r.MatchString("ab"))
a.IsFalse(r.MatchString("ABC"))
}
{
var r = re.MustCompile("(?i)abc|def|ghi")
a.IsTrue(r.MatchString("DEF"))
a.IsFalse(r.MatchString("ab"))
a.IsTrue(r.MatchString("ABC"))
}
}
@@ -46,39 +49,50 @@ func TestRegexp_Sub(t *testing.T) {
}
func TestRegexp_ParseKeywords(t *testing.T) {
var r = re.MustCompile("")
{
var keywords = r.ParseKeywords(`\n\t\n\f\r\v\x123`)
t.Log(keywords)
}
}
func TestRegexp_ParseKeywords2(t *testing.T) {
var a = assert.NewAssertion(t)
var r = re.MustCompile("")
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)def"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)|(?:def)"), []string{}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)|def"), []string{}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)def"), []string{"abcdef"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)|(?:def)"), []string{"abc", "def"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc)"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(abc|def|ghi)"), []string{"abc", "def", "ghi"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("(?i:abc)"), []string{}))
a.IsTrue(testCompareStrings(r.ParseKeywords("\babc"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(" \babc"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("\babc\b"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("\b(abc)"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`\babc`), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(` \babc`), []string{" "}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`\babc\b`), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`\b(abc)`), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc|efg|hij"), []string{"abc", "efg", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg*|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg?|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg+|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg{2,10}|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg{0,10}|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\|efg.+|hij"), []string{"abc", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("A(abc|bcd)"), []string{"abc", "bcd"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg|hij`), []string{"abc|efg", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg*|hij`), []string{"abc|ef", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg?|hij`), []string{"abc|ef", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg+|hij`), []string{"abc|ef", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg{2,10}|hij`), []string{"abc|ef", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg{0,10}|hij`), []string{"abc|ef", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc\|efg.+|hij`), []string{"abc|efg", "hij"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("A(abc|bcd)"), []string{"Aabc", "Abcd"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("^abc"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc$"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\$"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`abc$`), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc\\d"), []string{"abc"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("abc{0,4}"), []string{"ab"}))
a.IsTrue(testCompareStrings(r.ParseKeywords("{0,4}"), []string{}))
a.IsTrue(testCompareStrings(r.ParseKeywords("{1,4}"), []string{}))
a.IsTrue(testCompareStrings(r.ParseKeywords("中文|北京|上海|golang"), []string{"中文", "北京", "上海", "golang"}))
a.IsTrue(testCompareStrings(r.ParseKeywords(`(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick)\s*=`), strings.Split("onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick", "|")))
a.IsTrue(testCompareStrings(r.ParseKeywords(`/\*(!|\x00)`), []string{"/*"}))
}
func TestRegexp_ParseKeywords2(t *testing.T) {
func TestRegexp_ParseKeywords3(t *testing.T) {
var r = re.MustCompile("")
var policy = firewallconfigs.HTTPFirewallTemplate()
@@ -94,14 +108,23 @@ func TestRegexp_ParseKeywords2(t *testing.T) {
}
func BenchmarkRegexp_MatchString(b *testing.B) {
var r = re.MustCompile("(?i)abc|def|ghi")
var r = re.MustCompile("(?i)(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)(\\s|%09|%0A|(\\+|%20))*(=|%3D)")
//b.Log("keywords:", r.Keywords())
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}
}
func BenchmarkRegexp_MatchString2(b *testing.B) {
var r = regexp.MustCompile("(?i)abc|def|ghi")
var r = regexp.MustCompile("(?i)(onmouseover|onmousemove|onmousedown|onmouseup|onerror|onload|onclick|ondblclick|onkeydown|onkeyup|onkeypress)(\\s|%09|%0A|(\\+|%20))*(=|%3D)")
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}
}
func BenchmarkRegexp_MatchString_CaseSensitive(b *testing.B) {
var r = re.MustCompile("(abc|def|ghi)")
b.Log("keywords:", r.Keywords())
for i := 0; i < b.N; i++ {
r.MatchString("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
}

View File

@@ -25,6 +25,11 @@ func TestNewRuneTree(t *testing.T) {
a.IsTrue(tree.Lookup("iwind.liu@gmail.com", true))
}
func TestNewRuneTree2(t *testing.T) {
var tree = re.NewRuneTree([]string{"abc", "abd", "def", "GHI", "中国", "@"})
tree.Lookup("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36", true)
}
func BenchmarkRuneMap_Lookup(b *testing.B) {
var tree = re.NewRuneTree([]string{"abc", "abd", "def", "ghi", "中国"})
for i := 0; i < b.N; i++ {

View File

@@ -247,6 +247,12 @@ Loop:
if err != nil {
return err
}
// 正在退出时不上报错误
if teaconst.IsQuiting {
return nil
}
_, err = rpcClient.NodeLogRPC().CreateNodeLogs(rpcClient.Context(), &pb.CreateNodeLogsRequest{NodeLogs: logList})
return err
}

View File

@@ -15,6 +15,8 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/encoding/gzip"
"google.golang.org/grpc/metadata"
"net/url"
"sync"
@@ -137,6 +139,10 @@ func (this *RPCClient) FirewallService() pb.FirewallServiceClient {
return pb.NewFirewallServiceClient(this.pickConn())
}
func (this *RPCClient) SSLCertService() pb.SSLCertServiceClient {
return pb.NewSSLCertServiceClient(this.pickConn())
}
// Context 节点上下文信息
func (this *RPCClient) Context() context.Context {
ctx := context.Background()
@@ -214,12 +220,14 @@ func (this *RPCClient) init() error {
return errors.New("parse endpoint failed: " + err.Error())
}
var conn *grpc.ClientConn
var callOptions = grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(128*1024*1024),
grpc.UseCompressor(gzip.Name))
if u.Scheme == "http" {
conn, err = grpc.Dial(u.Host, grpc.WithInsecure(), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(128*1024*1024)))
conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(insecure.NewCredentials()))
} else if u.Scheme == "https" {
conn, err = grpc.Dial(u.Host, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(128*1024*1024)))
})), callOptions)
} else {
return errors.New("parse endpoint failed: invalid scheme '" + u.Scheme + "'")
}

View File

@@ -74,12 +74,12 @@ func (this *Cache) Write(key string, value interface{}, expiredAt int64) (ok boo
return
}
currentTimestamp := time.Now().Unix()
var currentTimestamp = utils.UnixTime()
if expiredAt <= currentTimestamp {
return
}
maxExpiredAt := currentTimestamp + 30*86400
var maxExpiredAt = currentTimestamp + 30*86400
if expiredAt > maxExpiredAt {
expiredAt = maxExpiredAt
}

View File

@@ -1,6 +1,7 @@
package ttlcache
import (
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
"github.com/iwind/TeaGo/rands"
"runtime"
"strconv"
@@ -29,19 +30,26 @@ func TestNewCache(t *testing.T) {
}
func TestCache_Memory(t *testing.T) {
var stat1 = &runtime.MemStats{}
runtime.ReadMemStats(stat1)
testutils.StartMemoryStats(t)
cache := NewCache()
for i := 0; i < 10_000_000; i++ {
var cache = NewCache()
var count = 2_000_000
for i := 0; i < count; i++ {
cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+3600)
}
t.Log("waiting ...")
var stat2 = &runtime.MemStats{}
runtime.ReadMemStats(stat2)
t.Log(cache.Count())
t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
time.Sleep(1 * time.Second)
for i := 0; i < count; i++ {
if i%2 == 0 {
cache.Delete("a" + strconv.Itoa(i))
}
}
t.Log(cache.Count())
cache.Count()
}
func BenchmarkCache_Add(b *testing.B) {

Some files were not shown because too many files have changed in this diff Show More