ChenKaiLiuG/GoEdgeNode
Archived
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

缓存策略实现LFU算法/实现内存缓存自动Flush数据到磁盘

Browse Source
This commit is contained in:
GoEdgeLab
2021-11-13 21:30:24 +08:00
parent 188fe12a0b
commit 07cb2bb303
20 changed files with 1100 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/caches/item.go
+26 -1
View File
@@ -2,6 +2,7 @@ package caches
import (
"github.com/TeaOSLab/EdgeNode/internal/utils"
"time"
)
type ItemType = int
@@ -11,6 +12,12 @@ const (
ItemTypeMemory ItemType = 2
)
// 计算当前周
// 不要用YW，因为需要计算两周是否临近
func currentWeek() int32 {
return int32(time.Now().Unix() / 604800)
}
type Item struct {
Type ItemType `json:"type"`
Key string `json:"key"`
@@ -20,6 +27,10 @@ type Item struct {
MetaSize int64 `json:"metaSize"`
Host string `json:"host"` // 主机名
ServerId int64 `json:"serverId"` // 服务ID
Week1Hits int64 `json:"week1Hits"`
Week2Hits int64 `json:"week2Hits"`
Week int32 `json:"week"`
}
func (this *Item) IsExpired() bool {
@@ -27,9 +38,23 @@ func (this *Item) IsExpired() bool {
}
func (this *Item) TotalSize() int64 {
return this.Size() + this.MetaSize + int64(len(this.Key)) + int64(len(this.Host)) + 64
return this.Size() + this.MetaSize + int64(len(this.Key)) + int64(len(this.Host))
}
func (this *Item) Size() int64 {
return this.HeaderSize + this.BodySize
}
func (this *Item) IncreaseHit(week int32) {
if this.Week == week {
this.Week2Hits++
} else {
if week-this.Week == 1 {
this.Week1Hits = this.Week2Hits
} else {
this.Week1Hits = 0
}
this.Week2Hits = 1
this.Week = week
}
}
internal/caches/item_test.go
+82
View File
@@ -0,0 +1,82 @@
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
package caches
import (
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"runtime"
"testing"
"time"
)
func TestItem_IncreaseHit(t *testing.T) {
var week = currentWeek()
var item = &Item{}
//item.Week = 2704
item.Week2Hits = 100
item.IncreaseHit(week)
t.Log("week:", item.Week, "week1:", item.Week1Hits, "week2:", item.Week2Hits)
item.IncreaseHit(week)
t.Log("week:", item.Week, "week1:", item.Week1Hits, "week2:", item.Week2Hits)
}
func TestItems_Memory(t *testing.T) {
var stat = &runtime.MemStats{}
runtime.ReadMemStats(stat)
var memory1 = stat.HeapInuse
var items = []*Item{}
for i := 0; i < 10_000_000; i++ {
items = append(items, &Item{
Key: types.String(i),
})
}
runtime.ReadMemStats(stat)
var memory2 = stat.HeapInuse
t.Log(memory1, memory2, (memory2-memory1)/1024/1024, "M")
var weekItems = make(map[string]*Item, 10_000_000)
for _, item := range items {
weekItems[item.Key] = item
}
runtime.ReadMemStats(stat)
var memory3 = stat.HeapInuse
t.Log(memory2, memory3, (memory3-memory2)/1024/1024, "M")
time.Sleep(1 * time.Second)
t.Log(len(items), len(weekItems))
}
func TestItems_Memory2(t *testing.T) {
var stat = &runtime.MemStats{}
runtime.ReadMemStats(stat)
var memory1 = stat.HeapInuse
var items = map[int32]map[string]bool{}
for i := 0; i < 10_000_000; i++ {
var week = int32((time.Now().Unix() - int64(86400*rands.Int(0, 300))) / (86400 * 7))
m, ok := items[week]
if !ok {
m = map[string]bool{}
items[week] = m
}
m[types.String(int64(i)*1_000_000)] = true
}
runtime.ReadMemStats(stat)
var memory2 = stat.HeapInuse
t.Log(memory1, memory2, (memory2-memory1)/1024/1024, "M")
time.Sleep(1 * time.Second)
for w, i := range items {
t.Log(w, len(i))
}
}
internal/caches/list_file.go
+135 -11
View File
@@ -8,6 +8,7 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/lists"
timeutil "github.com/iwind/TeaGo/utils/time"
_ "github.com/mattn/go-sqlite3"
"os"
"strconv"
@@ -24,6 +25,7 @@ type FileList struct {
onAdd func(item *Item)
onRemove func(item *Item)
// cacheItems
existsByHashStmt *sql.Stmt // 根据hash检查是否存在
insertStmt *sql.Stmt // 写入数据
selectByHashStmt *sql.Stmt // 使用hash查询数据
@@ -32,8 +34,15 @@ type FileList struct {
purgeStmt *sql.Stmt // 清理
deleteAllStmt *sql.Stmt // 删除所有数据
// hits
insertHitStmt *sql.Stmt // 写入数据
increaseHitStmt *sql.Stmt // 增加点击量
deleteHitByHashStmt *sql.Stmt // 根据hash删除数据
lfuHitsStmt *sql.Stmt // 读取老的数据
oldTables []string
itemsTableName string
hitsTableName string
isClosed bool
@@ -59,6 +68,7 @@ func (this *FileList) Init() error {
}
this.itemsTableName = "cacheItems_v2"
this.hitsTableName = "hits"
var dir = this.dir
if dir == "/" {
@@ -141,6 +151,23 @@ func (this *FileList) Init() error {
return err
}
this.insertHitStmt, err = this.db.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
this.increaseHitStmt, err = this.db.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
if err != nil {
return err
}
this.deleteHitByHashStmt, err = this.db.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
if err != nil {
return err
}
this.lfuHitsStmt, err = this.db.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
if err != nil {
return err
}
return nil
}
@@ -159,6 +186,11 @@ func (this *FileList) Add(hash string, item *Item) error {
return err
}
_, err = this.insertHitStmt.Exec(hash, timeutil.Format("YW"))
if err != nil {
return err
}
atomic.AddInt64(&this.total, 1)
if this.onAdd != nil {
@@ -253,6 +285,11 @@ func (this *FileList) Remove(hash string) error {
return err
}
_, err = this.deleteHitByHashStmt.Exec(hash)
if err != nil {
return err
}
atomic.AddInt64(&this.total, -1)
if this.onRemove != nil {
@@ -265,9 +302,9 @@ func (this *FileList) Remove(hash string) error {
// Purge 清理过期的缓存
// count 每次遍历的最大数量，控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用
func (this *FileList) Purge(count int, callback func(hash string) error) error {
func (this *FileList) Purge(count int, callback func(hash string) error) (int, error) {
if this.isClosed {
return nil
return 0, nil
}
if count <= 0 {
@@ -275,11 +312,56 @@ func (this *FileList) Purge(count int, callback func(hash string) error) error {
}
rows, err := this.purgeStmt.Query(time.Now().Unix(), count)
if err != nil {
return 0, err
}
hashStrings := []string{}
var countFound = 0
for rows.Next() {
var hash string
err = rows.Scan(&hash)
if err != nil {
_ = rows.Close()
return 0, err
}
hashStrings = append(hashStrings, hash)
countFound++
}
_ = rows.Close() // 不能使用defer，防止读写冲突
// 不在 rows.Next() 循环中操作是为了避免死锁
for _, hash := range hashStrings {
err = this.Remove(hash)
if err != nil {
return 0, err
}
err = callback(hash)
if err != nil {
return 0, err
}
}
return countFound, nil
}
func (this *FileList) PurgeLFU(count int, callback func(hash string) error) error {
if this.isClosed {
return nil
}
if count <= 0 {
return nil
}
rows, err := this.lfuHitsStmt.Query(count)
if err != nil {
return err
}
hashStrings := []string{}
var countFound = 0
for rows.Next() {
var hash string
err = rows.Scan(&hash)
@@ -288,6 +370,7 @@ func (this *FileList) Purge(count int, callback func(hash string) error) error {
return err
}
hashStrings = append(hashStrings, hash)
countFound++
}
_ = rows.Close() // 不能使用defer，防止读写冲突
@@ -303,7 +386,6 @@ func (this *FileList) Purge(count int, callback func(hash string) error) error {
return err
}
}
return nil
}
@@ -347,6 +429,13 @@ func (this *FileList) Count() (int64, error) {
return atomic.LoadInt64(&this.total), nil
}
// IncreaseHit 增加点击量
func (this *FileList) IncreaseHit(hash string) error {
var week = timeutil.Format("YW")
_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
return err
}
// OnAdd 添加事件
func (this *FileList) OnAdd(f func(item *Item)) {
this.onAdd = f
@@ -371,6 +460,11 @@ func (this *FileList) Close() error {
_ = this.purgeStmt.Close()
_ = this.deleteAllStmt.Close()
_ = this.insertHitStmt.Close()
_ = this.increaseHitStmt.Close()
_ = this.deleteHitByHashStmt.Close()
_ = this.lfuHitsStmt.Close()
return this.db.Close()
}
return nil
@@ -378,7 +472,8 @@ func (this *FileList) Close() error {
// 初始化
func (this *FileList) initTables(db *sql.DB, times int) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
{
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"key" varchar(1024),
@@ -411,17 +506,46 @@ ON "` + this.itemsTableName + `" (
"serverId" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := db.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(db, times+1)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := db.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
if dropErr == nil {
return this.initTables(db, times+1)
}
return err
}
return err
}
}
return err
{
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.hitsTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"hash" varchar(32),
"week1Hits" integer DEFAULT 0,
"week2Hits" integer DEFAULT 0,
"week" varchar(6)
);
CREATE UNIQUE INDEX IF NOT EXISTS "hits_hash"
ON "` + this.hitsTableName + `" (
"hash" ASC
);
`)
if err != nil {
// 尝试删除重建
if times < 3 {
_, dropErr := db.Exec(`DROP TABLE "` + this.hitsTableName + `"`)
if dropErr == nil {
return this.initTables(db, times+1)
}
return err
}
return err
}
}
return nil
internal/caches/list_file_test.go
+46 -1
View File
@@ -5,6 +5,7 @@ package caches
import (
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"strconv"
"sync"
@@ -184,7 +185,7 @@ func TestFileList_Purge(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = list.Purge(2, func(hash string) error {
_, err = list.Purge(2, func(hash string) error {
t.Log(hash)
return nil
})
@@ -257,6 +258,50 @@ func TestFileList_Conflict(t *testing.T) {
t.Log("after exists")
}
func TestFileList_IIF(t *testing.T) {
list := NewFileList(Tea.Root + "/data").(*FileList)
err := list.Init()
if err != nil {
t.Fatal(err)
}
rows, err := list.db.Query("SELECT IIF(0, 2, 3)")
if err != nil {
t.Fatal(err)
}
defer func() {
_ = rows.Close()
}()
if rows.Next() {
var result int
err = rows.Scan(&result)
if err != nil {
t.Fatal(err)
}
t.Log("result:", result)
}
}
func TestFileList_IncreaseHit(t *testing.T) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
if err != nil {
t.Fatal(err)
}
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
for i := 0; i < 1000_000; i++ {
err = list.IncreaseHit(stringutil.Md5("abc" + types.String(i)))
}
if err != nil {
t.Fatal(err)
}
t.Log("ok")
}
func BenchmarkFileList_Exist(b *testing.B) {
list := NewFileList(Tea.Root + "/data")
err := list.Init()
internal/caches/list_interface.go
+7 -1
View File
@@ -22,7 +22,10 @@ type ListInterface interface {
Remove(hash string) error
// Purge 清理过期数据
Purge(count int, callback func(hash string) error) error
Purge(count int, callback func(hash string) error) (int, error)
// PurgeLFU 清理LFU数据
PurgeLFU(count int, callback func(hash string) error) error
// CleanAll 清除所有缓存
CleanAll() error
@@ -41,4 +44,7 @@ type ListInterface interface {
// Close 关闭
Close() error
// IncreaseHit 增加点击量
IncreaseHit(hash string) error
}
internal/caches/list_memory.go
+175 -10
View File
@@ -5,12 +5,19 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
)
// MemoryList 内存缓存列表管理
type MemoryList struct {
count int64
itemMaps map[string]map[string]*Item // prefix => { hash => item }
weekItemMaps map[int32]map[string]bool // week => { hash => true }
minWeek int32
prefixes []string
locker sync.RWMutex
onAdd func(item *Item)
@@ -21,7 +28,9 @@ type MemoryList struct {
func NewMemoryList() ListInterface {
return &MemoryList{
itemMaps: map[string]map[string]*Item{},
itemMaps: map[string]map[string]*Item{},
weekItemMaps: map[int32]map[string]bool{},
minWeek: currentWeek(),
}
}
@@ -43,11 +52,19 @@ func (this *MemoryList) Reset() error {
for key := range this.itemMaps {
this.itemMaps[key] = map[string]*Item{}
}
this.weekItemMaps = map[int32]map[string]bool{}
this.locker.Unlock()
atomic.StoreInt64(&this.count, 0)
return nil
}
func (this *MemoryList) Add(hash string, item *Item) error {
if item.Week == 0 {
item.Week = currentWeek()
}
this.locker.Lock()
prefix := this.prefix(hash)
@@ -60,9 +77,20 @@ func (this *MemoryList) Add(hash string, item *Item) error {
// 先删除，为了可以正确触发统计
oldItem, ok := itemMap[hash]
if ok {
// 从week map中删除
if oldItem.Week > 0 {
wm, ok := this.weekItemMaps[oldItem.Week]
if ok {
delete(wm, hash)
}
}
// 回调
if this.onRemove != nil {
this.onRemove(oldItem)
}
} else {
atomic.AddInt64(&this.count, 1)
}
// 添加
@@ -71,6 +99,15 @@ func (this *MemoryList) Add(hash string, item *Item) error {
}
itemMap[hash] = item
// week map
wm, ok := this.weekItemMaps[item.Week]
if ok {
wm[hash] = true
} else {
this.weekItemMaps[item.Week] = map[string]bool{hash: true}
}
this.locker.Unlock()
return nil
}
@@ -122,7 +159,17 @@ func (this *MemoryList) Remove(hash string) error {
if this.onRemove != nil {
this.onRemove(item)
}
atomic.AddInt64(&this.count, -1)
delete(itemMap, hash)
// week map
if item.Week > 0 {
wm, ok := this.weekItemMaps[item.Week]
if ok {
delete(wm, hash)
}
}
}
this.locker.Unlock()
@@ -132,7 +179,7 @@ func (this *MemoryList) Remove(hash string) error {
// Purge 清理过期的缓存
// count 每次遍历的最大数量，控制此数字可以保证每次清理的时候不用花太多时间
// callback 每次发现过期key的调用
func (this *MemoryList) Purge(count int, callback func(hash string) error) error {
func (this *MemoryList) Purge(count int, callback func(hash string) error) (int, error) {
this.locker.Lock()
deletedHashList := []string{}
@@ -146,8 +193,9 @@ func (this *MemoryList) Purge(count int, callback func(hash string) error) error
itemMap, ok := this.itemMaps[prefix]
if !ok {
this.locker.Unlock()
return nil
return 0, nil
}
var countFound = 0
for hash, item := range itemMap {
if count <= 0 {
break
@@ -157,14 +205,100 @@ func (this *MemoryList) Purge(count int, callback func(hash string) error) error
if this.onRemove != nil {
this.onRemove(item)
}
atomic.AddInt64(&this.count, -1)
delete(itemMap, hash)
deletedHashList = append(deletedHashList, hash)
// week map
if item.Week > 0 {
wm, ok := this.weekItemMaps[item.Week]
if ok {
delete(wm, hash)
}
}
countFound++
}
count--
}
this.locker.Unlock()
// 执行外部操作
for _, hash := range deletedHashList {
if callback != nil {
err := callback(hash)
if err != nil {
return 0, err
}
}
}
return countFound, nil
}
func (this *MemoryList) PurgeLFU(count int, callback func(hash string) error) error {
if count <= 0 {
return nil
}
var week = currentWeek()
if this.minWeek > week {
this.minWeek = week
}
var deletedHashList = []string{}
Loop:
for w := this.minWeek; w <= week; w++ {
this.minWeek = w
this.locker.Lock()
wm, ok := this.weekItemMaps[w]
if ok {
var wc = len(wm)
if wc == 0 {
delete(this.weekItemMaps, w)
} else {
if wc <= count {
delete(this.weekItemMaps, w)
}
// TODO 未来支持按照点击量排序
for hash := range wm {
count--
if count < 0 {
this.locker.Unlock()
break Loop
}
delete(wm, hash)
itemMap, ok := this.itemMaps[this.prefix(hash)]
if !ok {
continue
}
item, ok := itemMap[hash]
if !ok {
continue
}
if this.onRemove != nil {
this.onRemove(item)
}
atomic.AddInt64(&this.count, -1)
delete(itemMap, hash)
deletedHashList = append(deletedHashList, hash)
}
}
} else {
delete(this.weekItemMaps, w)
}
this.locker.Unlock()
}
// 执行外部操作
for _, hash := range deletedHashList {
if callback != nil {
@@ -174,6 +308,7 @@ func (this *MemoryList) Purge(count int, callback func(hash string) error) error
}
}
}
return nil
}
@@ -206,13 +341,8 @@ func (this *MemoryList) Stat(check func(hash string) bool) (*Stat, error) {
// Count 总数量
func (this *MemoryList) Count() (int64, error) {
this.locker.RLock()
var count = 0
for _, itemMap := range this.itemMaps {
count += len(itemMap)
}
this.locker.RUnlock()
return int64(count), nil
var count = atomic.LoadInt64(&this.count)
return count, nil
}
// OnAdd 添加事件
@@ -229,6 +359,41 @@ func (this *MemoryList) Close() error {
return nil
}
// IncreaseHit 增加点击量
func (this *MemoryList) IncreaseHit(hash string) error {
this.locker.Lock()
itemMap, ok := this.itemMaps[this.prefix(hash)]
if !ok {
this.locker.Unlock()
return nil
}
item, ok := itemMap[hash]
if ok {
var week = currentWeek()
// 交换位置
if item.Week > 0 && item.Week != week {
wm, ok := this.weekItemMaps[item.Week]
if ok {
delete(wm, hash)
}
wm, ok = this.weekItemMaps[week]
if ok {
wm[hash] = true
} else {
this.weekItemMaps[week] = map[string]bool{hash: true}
}
}
item.IncreaseHit(week)
}
this.locker.Unlock()
return nil
}
func (this *MemoryList) print(t *testing.T) {
this.locker.Lock()
for _, itemMap := range this.itemMaps {
internal/caches/list_memory_test.go
+78 -5
View File
@@ -31,6 +31,8 @@ func TestMemoryList_Add(t *testing.T) {
})
t.Log(list.prefixes)
logs.PrintAsJSON(list.itemMaps, t)
logs.PrintAsJSON(list.weekItemMaps, t)
t.Log(list.Count())
}
func TestMemoryList_Remove(t *testing.T) {
@@ -48,6 +50,8 @@ func TestMemoryList_Remove(t *testing.T) {
})
_ = list.Remove("b")
list.print(t)
logs.PrintAsJSON(list.weekItemMaps, t)
t.Log(list.Count())
}
func TestMemoryList_Purge(t *testing.T) {
@@ -73,19 +77,22 @@ func TestMemoryList_Purge(t *testing.T) {
ExpiredAt: time.Now().Unix() - 2,
HeaderSize: 1024,
})
_ = list.Purge(100, func(hash string) error {
_, _ = list.Purge(100, func(hash string) error {
t.Log("delete:", hash)
return nil
})
list.print(t)
logs.PrintAsJSON(list.weekItemMaps, t)
for i := 0; i < 1000; i++ {
_ = list.Purge(100, func(hash string) error {
_, _ = list.Purge(100, func(hash string) error {
t.Log("delete:", hash)
return nil
})
t.Log(list.purgeIndex)
}
t.Log(list.Count())
}
func TestMemoryList_Purge_Large_List(t *testing.T) {
@@ -139,7 +146,7 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
_ = list.Init()
before := time.Now()
for i := 0; i < 1_000_000; i++ {
key := "http://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
key := "https://www.teaos.cn/hello/" + strconv.Itoa(i/10000) + "/" + strconv.Itoa(i) + ".html"
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
Key: key,
ExpiredAt: time.Now().Unix() + 3600,
@@ -150,7 +157,7 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
t.Log(time.Since(before).Seconds()*1000, "ms")
before = time.Now()
err := list.CleanPrefix("http://www.teaos.cn/hello/10")
err := list.CleanPrefix("https://www.teaos.cn/hello/10")
if err != nil {
t.Fatal(err)
}
@@ -162,11 +169,77 @@ func TestMemoryList_CleanPrefix(t *testing.T) {
t.Log(time.Since(before).Seconds()*1000, "ms")
}
func TestMemoryList_PurgeLFU(t *testing.T) {
var list = NewMemoryList().(*MemoryList)
list.minWeek = 2704
var before = time.Now()
defer func() {
t.Log(time.Since(before).Seconds()*1000, "ms")
}()
t.Log("current week:", currentWeek())
_ = list.Add("1", &Item{})
_ = list.Add("2", &Item{})
_ = list.Add("3", &Item{})
_ = list.Add("4", &Item{})
_ = list.Add("5", &Item{})
_ = list.Add("6", &Item{Week: 2704})
_ = list.Add("7", &Item{Week: 2704})
_ = list.Add("8", &Item{Week: 2705})
err := list.PurgeLFU(2, func(hash string) error {
t.Log("purge lfu:", hash)
return nil
})
if err != nil {
t.Fatal(err)
}
t.Log("ok")
logs.PrintAsJSON(list.weekItemMaps, t)
t.Log(list.Count())
}
func TestMemoryList_IncreaseHit(t *testing.T) {
var list = NewMemoryList().(*MemoryList)
var item = &Item{}
item.Week = 2705
item.Week2Hits = 100
_ = list.Add("a", &Item{})
_ = list.Add("a", item)
t.Log("hits1:", item.Week1Hits, "hits2:", item.Week2Hits, "week:", item.Week)
logs.PrintAsJSON(list.weekItemMaps, t)
_ = list.IncreaseHit("a")
t.Log("hits1:", item.Week1Hits, "hits2:", item.Week2Hits, "week:", item.Week)
logs.PrintAsJSON(list.weekItemMaps, t)
_ = list.IncreaseHit("a")
t.Log("hits1:", item.Week1Hits, "hits2:", item.Week2Hits, "week:", item.Week)
logs.PrintAsJSON(list.weekItemMaps, t)
}
func TestMemoryList_CleanAll(t *testing.T) {
var list = NewMemoryList().(*MemoryList)
var item = &Item{}
item.Week = 2705
item.Week2Hits = 100
_ = list.Add("a", &Item{})
_ = list.CleanAll()
logs.PrintAsJSON(list.itemMaps, t)
logs.PrintAsJSON(list.weekItemMaps, t)
t.Log(list.Count())
}
func TestMemoryList_GC(t *testing.T) {
list := NewMemoryList().(*MemoryList)
_ = list.Init()
for i := 0; i < 1_000_000; i++ {
key := "http://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
key := "https://www.teaos.cn/hello" + strconv.Itoa(i/100000) + "/" + strconv.Itoa(i) + ".html"
_ = list.Add(fmt.Sprintf("%d", xxhash.Sum64String(key)), &Item{
Key: key,
ExpiredAt: 0,
internal/caches/manager.go
+1 -1
View File
@@ -135,7 +135,7 @@ func (this *Manager) NewStorageWithPolicy(policy *serverconfigs.HTTPCachePolicy)
case serverconfigs.CachePolicyStorageFile:
return NewFileStorage(policy)
case serverconfigs.CachePolicyStorageMemory:
return NewMemoryStorage(policy)
return NewMemoryStorage(policy, nil)
}
return nil
}
internal/caches/storage_file.go
+108 -12
View File
@@ -11,10 +11,13 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/iwind/TeaGo/Tea"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
stringutil "github.com/iwind/TeaGo/utils/string"
"golang.org/x/text/language"
"golang.org/x/text/message"
"io"
"math"
"os"
"path/filepath"
"regexp"
@@ -165,12 +168,16 @@ func (this *FileStorage) Init() error {
Life: this.policy.Life,
MinLife: this.policy.MinLife,
MaxLife: this.policy.MaxLife,
MemoryAutoPurgeCount: this.policy.MemoryAutoPurgeCount,
MemoryAutoPurgeInterval: this.policy.MemoryAutoPurgeInterval,
MemoryLFUFreePercent: this.policy.MemoryLFUFreePercent,
}
err = memoryPolicy.Init()
if err != nil {
return err
}
memoryStorage := NewMemoryStorage(memoryPolicy)
memoryStorage := NewMemoryStorage(memoryPolicy, this)
err = memoryStorage.Init()
if err != nil {
return err
@@ -227,6 +234,17 @@ func (this *FileStorage) OpenReader(key string) (Reader, error) {
return nil, err
}
// 增加点击量
// 1/1000采样
// TODO 考虑是否在缓存策略里设置
if rands.Int(0, 1000) == 0 {
var hitErr = this.list.IncreaseHit(hash)
if hitErr != nil {
// 此错误可以忽略
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
}
}
isOk = true
return reader, nil
}
@@ -398,7 +416,7 @@ func (this *FileStorage) AddToList(item *Item) {
}
}
item.MetaSize = SizeMeta
item.MetaSize = SizeMeta + 128
hash := stringutil.Md5(item.Key)
err := this.list.Add(hash, item)
if err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
@@ -619,7 +637,14 @@ func (this *FileStorage) initList() error {
}()
// 启动定时清理任务
this.ticker = utils.NewTicker(30 * time.Second)
var autoPurgeInterval = this.policy.PersistenceAutoPurgeInterval
if autoPurgeInterval <= 0 {
autoPurgeInterval = 30
if Tea.IsTesting() {
autoPurgeInterval = 10
}
}
this.ticker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
events.On(events.EventQuit, func() {
remotelogs.Println("CACHE", "quit clean timer")
var ticker = this.ticker
@@ -730,16 +755,87 @@ func (this *FileStorage) decodeFile(path string) (*Item, error) {
// 清理任务
func (this *FileStorage) purgeLoop() {
err := this.list.Purge(1000, func(hash string) error {
path := this.hashPath(hash)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
// 计算是否应该开启LFU清理
var capacityBytes = this.policy.CapacityBytes()
var startLFU = false
var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
var lfuFreePercent = this.policy.PersistenceLFUFreePercent
if lfuFreePercent <= 0 {
lfuFreePercent = 5
}
if capacityBytes > 0 {
if lfuFreePercent < 100 {
if usedPercent >= 100-lfuFreePercent {
startLFU = true
}
}
}
// 清理过期
{
var times = 1
// 空闲时间多清理
if utils.SharedFreeHoursManager.IsFreeHour() {
times = 5
}
// 处于LFU阈值时，多清理
if startLFU {
times = 5
}
var purgeCount = this.policy.PersistenceAutoPurgeCount
if purgeCount <= 0 {
purgeCount = 1000
}
for i := 0; i < times; i++ {
countFound, err := this.list.Purge(purgeCount, func(hash string) error {
path := this.hashPath(hash)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge file storage failed: "+err.Error())
continue
}
if countFound < purgeCount {
break
}
time.Sleep(1 * time.Second)
}
}
// 磁盘空间不足时，清除老旧的缓存
if startLFU {
var total, _ = this.list.Count()
if total > 0 {
var count = types.Int(math.Ceil(float64(total) * float64(lfuFreePercent*2) / 100))
if count > 0 {
// 限制单次清理的条数，防止占用太多系统资源
if count > 2000 {
count = 2000
}
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
err := this.list.PurgeLFU(count, func(hash string) error {
path := this.hashPath(hash)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge file storage in LFU failed: "+err.Error())
}
}
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge file storage failed: "+err.Error())
}
}
internal/caches/storage_memory.go
+165 -16
View File
@@ -6,6 +6,9 @@ import (
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
"github.com/TeaOSLab/EdgeNode/internal/utils"
"github.com/cespare/xxhash"
"github.com/iwind/TeaGo/rands"
"github.com/iwind/TeaGo/types"
"math"
"strconv"
"sync"
"sync/atomic"
@@ -26,22 +29,37 @@ func (this *MemoryItem) IsExpired() bool {
}
type MemoryStorage struct {
policy *serverconfigs.HTTPCachePolicy
list ListInterface
locker *sync.RWMutex
valuesMap map[uint64]*MemoryItem
ticker *utils.Ticker
purgeDuration time.Duration
parentStorage StorageInterface
policy *serverconfigs.HTTPCachePolicy
list ListInterface
locker *sync.RWMutex
valuesMap map[uint64]*MemoryItem // hash => item
dirtyChan chan string // hash chan
purgeTicker *utils.Ticker
totalSize int64
writingKeyMap map[string]bool // key => bool
}
func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy) *MemoryStorage {
func NewMemoryStorage(policy *serverconfigs.HTTPCachePolicy, parentStorage StorageInterface) *MemoryStorage {
var dirtyChan chan string
if parentStorage != nil {
var queueSize = policy.MemoryAutoFlushQueueSize
if queueSize <= 0 {
queueSize = 2048
}
dirtyChan = make(chan string, queueSize)
}
return &MemoryStorage{
parentStorage: parentStorage,
policy: policy,
list: NewMemoryList(),
locker: &sync.RWMutex{},
valuesMap: map[uint64]*MemoryItem{},
dirtyChan: dirtyChan,
writingKeyMap: map[string]bool{},
}
}
@@ -57,18 +75,26 @@ func (this *MemoryStorage) Init() error {
atomic.AddInt64(&this.totalSize, -item.TotalSize())
})
if this.purgeDuration <= 0 {
this.purgeDuration = 10 * time.Second
var autoPurgeInterval = this.policy.MemoryAutoPurgeInterval
if autoPurgeInterval <= 0 {
autoPurgeInterval = 5
}
// 启动定时清理任务
this.ticker = utils.NewTicker(this.purgeDuration)
this.purgeTicker = utils.NewTicker(time.Duration(autoPurgeInterval) * time.Second)
go func() {
for this.ticker.Next() {
for this.purgeTicker.Next() {
this.purgeLoop()
}
}()
// 启动定时Flush memory to disk任务
go func() {
for hash := range this.dirtyChan {
this.flushItem(hash)
}
}()
return nil
}
@@ -91,6 +117,18 @@ func (this *MemoryStorage) OpenReader(key string) (Reader, error) {
return nil, err
}
this.locker.RUnlock()
// 增加点击量
// 1/1000采样
// TODO 考虑是否在缓存策略里设置
if rands.Int(0, 1000) == 0 {
var hitErr = this.list.IncreaseHit(types.String(hash))
if hitErr != nil {
// 此错误可以忽略
remotelogs.Error("CACHE", "increase hit failed: "+hitErr.Error())
}
}
return reader, nil
}
this.locker.RUnlock()
@@ -145,7 +183,7 @@ func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int) (
}
isWriting = true
return NewMemoryWriter(this.valuesMap, key, expiredAt, status, this.locker, func() {
return NewMemoryWriter(this, key, expiredAt, status, func() {
this.locker.Lock()
delete(this.writingKeyMap, key)
this.locker.Unlock()
@@ -210,8 +248,12 @@ func (this *MemoryStorage) Stop() {
this.valuesMap = map[uint64]*MemoryItem{}
this.writingKeyMap = map[string]bool{}
_ = this.list.Reset()
if this.ticker != nil {
this.ticker.Stop()
if this.purgeTicker != nil {
this.purgeTicker.Stop()
}
if this.parentStorage != nil && this.dirtyChan != nil {
close(this.dirtyChan)
}
_ = this.list.Close()
@@ -228,7 +270,7 @@ func (this *MemoryStorage) Policy() *serverconfigs.HTTPCachePolicy {
// AddToList 将缓存添加到列表
func (this *MemoryStorage) AddToList(item *Item) {
item.MetaSize = int64(len(item.Key)) + 32 /** 32是我们评估的数据结构的长度 **/
item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/
hash := fmt.Sprintf("%d", this.hash(item.Key))
_ = this.list.Add(hash, item)
}
@@ -250,7 +292,28 @@ func (this *MemoryStorage) hash(key string) uint64 {
// 清理任务
func (this *MemoryStorage) purgeLoop() {
_ = this.list.Purge(2048, func(hash string) error {
// 计算是否应该开启LFU清理
var capacityBytes = this.policy.CapacityBytes()
var startLFU = false
var usedPercent = float32(this.TotalMemorySize()*100) / float32(capacityBytes)
var lfuFreePercent = this.policy.MemoryLFUFreePercent
if lfuFreePercent <= 0 {
lfuFreePercent = 5
}
if capacityBytes > 0 {
if lfuFreePercent < 100 {
if usedPercent >= 100-lfuFreePercent {
startLFU = true
}
}
}
// 清理过期
var purgeCount = this.policy.MemoryAutoPurgeCount
if purgeCount <= 0 {
purgeCount = 2000
}
_, _ = this.list.Purge(purgeCount, func(hash string) error {
uintHash, err := strconv.ParseUint(hash, 10, 64)
if err == nil {
this.locker.Lock()
@@ -259,6 +322,92 @@ func (this *MemoryStorage) purgeLoop() {
}
return nil
})
// LFU
if startLFU {
var total, _ = this.list.Count()
if total > 0 {
var count = types.Int(math.Ceil(float64(total) * float64(lfuFreePercent * 2) / 100))
if count > 0 {
// 限制单次清理的条数，防止占用太多系统资源
if count > 2000 {
count = 2000
}
remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
err := this.list.PurgeLFU(count, func(hash string) error {
uintHash, err := strconv.ParseUint(hash, 10, 64)
if err == nil {
this.locker.Lock()
delete(this.valuesMap, uintHash)
this.locker.Unlock()
}
return nil
})
if err != nil {
remotelogs.Warn("CACHE", "purge memory storage in LFU failed: "+err.Error())
}
}
}
}
}
// Flush任务
func (this *MemoryStorage) flushItem(key string) {
if this.parentStorage == nil {
return
}
var hash = this.hash(key)
this.locker.RLock()
item, ok := this.valuesMap[hash]
this.locker.RUnlock()
if !ok {
return
}
if !item.IsDone || item.IsExpired() {
return
}
writer, err := this.parentStorage.OpenWriter(key, item.ExpiredAt, item.Status)
if err != nil {
if !CanIgnoreErr(err) {
remotelogs.Error("CACHE", "flush items failed: open writer failed: "+err.Error())
}
return
}
_, err = writer.WriteHeader(item.HeaderValue)
if err != nil {
_ = writer.Discard()
remotelogs.Error("CACHE", "flush items failed: write header failed: "+err.Error())
return
}
_, err = writer.Write(item.BodyValue)
if err != nil {
_ = writer.Discard()
remotelogs.Error("CACHE", "flush items failed: writer body failed: "+err.Error())
return
}
err = writer.Close()
if err != nil {
_ = writer.Discard()
remotelogs.Error("CACHE", "flush items failed: close writer failed: "+err.Error())
}
this.parentStorage.AddToList(&Item{
Type: writer.ItemType(),
Key: key,
ExpiredAt: item.ExpiredAt,
HeaderSize: writer.HeaderSize(),
BodySize: writer.BodySize(),
})
return
}
func (this *MemoryStorage) memoryCapacityBytes() int64 {
internal/caches/storage_memory_test.go
+3 -2
View File
@@ -230,8 +230,9 @@ func TestMemoryStorage_Purge(t *testing.T) {
}
func TestMemoryStorage_Expire(t *testing.T) {
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{})
storage.purgeDuration = 5 * time.Second
storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
MemoryAutoPurgeInterval: 5,
})
err := storage.Init()
if err != nil {
t.Fatal(err)
internal/caches/writer_memory.go
+17 -12
View File
@@ -2,15 +2,14 @@ package caches
import (
"github.com/cespare/xxhash"
"sync"
"time"
)
type MemoryWriter struct {
storage *MemoryStorage
key string
expiredAt int64
m map[uint64]*MemoryItem
locker *sync.RWMutex
headerSize int64
bodySize int64
status int
@@ -20,12 +19,11 @@ type MemoryWriter struct {
endFunc func()
}
func NewMemoryWriter(m map[uint64]*MemoryItem, key string, expiredAt int64, status int, locker *sync.RWMutex, endFunc func()) *MemoryWriter {
func NewMemoryWriter(memoryStorage *MemoryStorage, key string, expiredAt int64, status int, endFunc func()) *MemoryWriter {
w := &MemoryWriter{
m: m,
storage: memoryStorage,
key: key,
expiredAt: expiredAt,
locker: locker,
item: &MemoryItem{
ExpiredAt: expiredAt,
ModifiedAt: time.Now().Unix(),
@@ -72,10 +70,17 @@ func (this *MemoryWriter) Close() error {
return nil
}
this.locker.Lock()
this.storage.locker.Lock()
this.item.IsDone = true
this.m[this.hash] = this.item
this.locker.Unlock()
this.storage.valuesMap[this.hash] = this.item
if this.storage.parentStorage != nil {
select {
case this.storage.dirtyChan <- this.key:
default:
}
}
this.storage.locker.Unlock()
return nil
}
@@ -85,9 +90,9 @@ func (this *MemoryWriter) Discard() error {
// 需要在Locker之外
defer this.endFunc()
this.locker.Lock()
delete(this.m, this.hash)
this.locker.Unlock()
this.storage.locker.Lock()
delete(this.storage.valuesMap, this.hash)
this.storage.locker.Unlock()
return nil
}