通过内存缓存提升文件缓存效率大约20%

internal/caches/list_file.go

@@ -5,6 +5,7 @@ package caches
 import (
 	"database/sql"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/iwind/TeaGo/lists"
 	_ "github.com/mattn/go-sqlite3"
@@ -35,10 +36,15 @@ type FileList struct {
 	itemsTableName string
 
 	isClosed bool
+
+	memoryCache *ttlcache.Cache
 }
 
 func NewFileList(dir string) ListInterface {
-	return &FileList{dir: dir}
+	return &FileList{
+		dir:         dir,
+		memoryCache: ttlcache.NewCache(),
+	}
 }
 
 func (this *FileList) Init() error {
@@ -100,7 +106,7 @@ func (this *FileList) Init() error {
 	this.total = total
 
 	// 常用语句
-	this.existsByHashStmt, err = this.db.Prepare(`SELECT "bodySize" FROM "` + this.itemsTableName + `" WHERE "hash"=? AND expiredAt>? LIMIT 1`)
+	this.existsByHashStmt, err = this.db.Prepare(`SELECT "expiredAt" FROM "` + this.itemsTableName + `" WHERE "hash"=? AND expiredAt>? LIMIT 1`)
 	if err != nil {
 		return err
 	}
@@ -166,6 +172,11 @@ func (this *FileList) Exist(hash string) (bool, error) {
 		return false, nil
 	}
 
+	item := this.memoryCache.Read(hash)
+	if item != nil {
+		return true, nil
+	}
+
 	rows, err := this.existsByHashStmt.Query(hash, time.Now().Unix())
 	if err != nil {
 		return false, err
@@ -174,6 +185,12 @@ func (this *FileList) Exist(hash string) (bool, error) {
 		_ = rows.Close()
 	}()
 	if rows.Next() {
+		var expiredAt int64
+		err = rows.Scan(&expiredAt)
+		if err != nil {
+			return true, nil
+		}
+		this.memoryCache.Write(hash, 1, expiredAt)
 		return true, nil
 	}
 	return false, nil
@@ -210,6 +227,9 @@ func (this *FileList) Remove(hash string) error {
 		return nil
 	}
 
+	// 从缓存中删除
+	this.memoryCache.Delete(hash)
+
 	row := this.selectByHashStmt.QueryRow(hash)
 	if row.Err() != nil {
 		return row.Err()
@@ -288,6 +308,8 @@ func (this *FileList) CleanAll() error {
 		return nil
 	}
 
+	this.memoryCache.Clean()
+
 	_, err := this.deleteAllStmt.Exec()
 	if err != nil {
 		return err

internal/caches/list_file_test.go

@@ -50,7 +50,7 @@ func TestFileList_Add_Many(t *testing.T) {
 	}
 	before := time.Now()
 	for i := 0; i < 2000_0000; i++ {
-		u := "http://edge.teaos.cn/123456" + strconv.Itoa(i)
+		u := "https://edge.teaos.cn/123456" + strconv.Itoa(i)
 		_ = list.Add(stringutil.Md5(u), &Item{
 			Key:        u,
 			ExpiredAt:  time.Now().Unix() + 3600,

internal/ttlcache/cache.go

@@ -5,7 +5,7 @@ import (
 	"time"
 )
 
-// TTL缓存
+// Cache TTL缓存
 // 最大的缓存时间为30 * 86400
 // Piece数据结构：
 //      Piece1          |  Piece2 | Piece3 | ...
@@ -136,6 +136,12 @@ func (this *Cache) GC() {
 	this.gcPieceIndex = newIndex
 }
 
+func (this *Cache) Clean() {
+	for _, piece := range this.pieces {
+		piece.Clean()
+	}
+}
+
 func (this *Cache) Destroy() {
 	this.isDestroyed = true
 

internal/ttlcache/piece.go

@@ -17,7 +17,7 @@ func NewPiece(maxItems int) *Piece {
 	return &Piece{m: map[uint64]*Item{}, maxItems: maxItems}
 }
 
-func (this *Piece) Add(key uint64, item *Item) () {
+func (this *Piece) Add(key uint64, item *Item) {
 	this.locker.Lock()
 	if len(this.m) >= this.maxItems {
 		this.locker.Unlock()
@@ -82,6 +82,12 @@ func (this *Piece) GC() {
 	this.locker.Unlock()
 }
 
+func (this *Piece) Clean() {
+	this.locker.Lock()
+	this.m = map[uint64]*Item{}
+	this.locker.Unlock()
+}
+
 func (this *Piece) Destroy() {
 	this.locker.Lock()
 	this.m = nil