优化代码

build/build.sh

@@ -50,9 +50,9 @@ function build() {
 	fi
 
 	cp "$ROOT"/configs/api.template.yaml "$DIST"/configs
+	cp "$ROOT"/configs/cluster.template.yaml "$DIST"/configs
 	cp -R "$ROOT"/www "$DIST"/
 	cp -R "$ROOT"/pages "$DIST"/
-	cp -R "$ROOT"/resources "$DIST"/
 
 	# we support TOA on linux/amd64 only
 	if [ "$OS" == "linux" -a "$ARCH" == "amd64" ]

cmd/edge-node/main.go

@@ -25,7 +25,7 @@ func main() {
 		Product(teaconst.ProductName).
 		Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|pprof|accesslog]").
 		Usage(teaconst.ProcessName + " [trackers|goman|conns|gc]").
-		Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove] IP")
+		Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove|ip.close] IP")
 
 	app.On("test", func() {
 		err := nodes.NewNode().Test()
@@ -241,6 +241,38 @@ func main() {
 			}
 		}
 	})
+	app.On("ip.close", func() {
+		var args = os.Args[2:]
+		if len(args) == 0 {
+			fmt.Println("Usage: edge-node ip.close IP")
+			return
+		}
+		var ip = args[0]
+		if len(net.ParseIP(ip)) == 0 {
+			fmt.Println("IP '" + ip + "' is invalid")
+			return
+		}
+
+		fmt.Println("close ip '" + ip)
+
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{
+			Code: "closeIP",
+			Params: map[string]any{
+				"ip": ip,
+			},
+		})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+		} else {
+			var errString = maps.NewMap(reply.Params).GetString("error")
+			if len(errString) > 0 {
+				fmt.Println("[ERROR]" + errString)
+			} else {
+				fmt.Println("ok")
+			}
+		}
+	})
 	app.On("ip.remove", func() {
 		var args = os.Args[2:]
 		if len(args) == 0 {
@@ -318,6 +350,21 @@ func main() {
 			}
 		}
 	})
+	app.On("bandwidth", func() {
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{Code: "bandwidth"})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+			return
+		}
+		var statsMap = maps.NewMap(reply.Params).Get("stats")
+		statsJSON, err := json.MarshalIndent(statsMap, "", "  ")
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+			return
+		}
+		fmt.Println(string(statsJSON))
+	})
 	app.Run(func() {
 		var node = nodes.NewNode()
 		node.Start()

internal/apps/log_writer.go

@@ -41,7 +41,7 @@ func (this *LogWriter) Init() {
 	this.c = make(chan string, 1024)
 
 	// 异步写入文件
-	var maxFileSize = 2 * sizes.G // 文件最大尺寸，超出此尺寸则清空
+	var maxFileSize = 128 * sizes.M // 文件最大尺寸，超出此尺寸则清空
 	if fp != nil {
 		goman.New(func() {
 			var totalSize int64 = 0

internal/caches/consts.go

@@ -3,6 +3,7 @@
 package caches
 
 const (
+	SuffixAll         = "@GOEDGE_"        // 通用后缀
 	SuffixWebP        = "@GOEDGE_WEBP"    // WebP后缀
 	SuffixCompression = "@GOEDGE_"        // 压缩后缀 SuffixCompression + Encoding
 	SuffixMethod      = "@GOEDGE_"        // 请求方法后缀 SuffixMethod + RequestMethod

internal/caches/file_dir.go

@@ -0,0 +1,11 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import "github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+
+type FileDir struct {
+	Path     string
+	Capacity *shared.SizeCapacity
+	IsFull   bool
+}

internal/caches/item.go

@@ -2,6 +2,7 @@ package caches
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"strings"
 	"time"
 )
 
@@ -59,3 +60,17 @@ func (this *Item) IncreaseHit(week int32) {
 		this.Week = week
 	}
 }
+
+func (this *Item) RequestURI() string {
+	var schemeIndex = strings.Index(this.Key, "://")
+	if schemeIndex <= 0 {
+		return ""
+	}
+
+	var firstSlashIndex = strings.Index(this.Key[schemeIndex+3:], "/")
+	if firstSlashIndex <= 0 {
+		return ""
+	}
+
+	return this.Key[schemeIndex+3+firstSlashIndex:]
+}

internal/caches/item_test.go

@@ -81,3 +81,14 @@ func TestItems_Memory2(t *testing.T) {
 		t.Log(w, len(i))
 	}
 }
+
+func TestItem_RequestURI(t *testing.T) {
+	for _, u := range []string{
+		"https://goedge.cn/hello/world",
+		"https://goedge.cn:8080/hello/world",
+		"https://goedge.cn/hello/world?v=1&t=123",
+	} {
+		var item = &Item{Key: u}
+		t.Log(u, "=>", item.RequestURI())
+	}
+}

internal/caches/list_file.go

@@ -96,7 +96,7 @@ func (this *FileList) Reset() error {
 }
 
 func (this *FileList) Add(hash string, item *Item) error {
-	var db = this.getDB(hash)
+	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
 		return nil
@@ -120,12 +120,17 @@ func (this *FileList) Add(hash string, item *Item) error {
 }
 
 func (this *FileList) Exist(hash string) (bool, error) {
-	var db = this.getDB(hash)
+	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
 		return false, nil
 	}
 
+	// 如果Hash列表里不存在，那么必然不存在
+	if !db.hashMap.Exist(hash) {
+		return false, nil
+	}
+
 	var item = this.memoryCache.Read(hash)
 	if item != nil {
 		return true, nil
@@ -155,6 +160,7 @@ func (this *FileList) CleanPrefix(prefix string) error {
 	}
 
 	defer func() {
+		// TODO 需要优化
 		this.memoryCache.Clean()
 	}()
 
@@ -167,6 +173,46 @@ func (this *FileList) CleanPrefix(prefix string) error {
 	return nil
 }
 
+// CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello
+func (this *FileList) CleanMatchKey(key string) error {
+	if len(key) == 0 {
+		return nil
+	}
+
+	defer func() {
+		// TODO 需要优化
+		this.memoryCache.Clean()
+	}()
+
+	for _, db := range this.dbList {
+		err := db.CleanMatchKey(key)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/
+func (this *FileList) CleanMatchPrefix(prefix string) error {
+	if len(prefix) == 0 {
+		return nil
+	}
+
+	defer func() {
+		// TODO 需要优化
+		this.memoryCache.Clean()
+	}()
+
+	for _, db := range this.dbList {
+		err := db.CleanMatchPrefix(prefix)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (this *FileList) Remove(hash string) error {
 	_, err := this.remove(hash)
 	return err
@@ -225,7 +271,7 @@ func (this *FileList) PurgeLFU(count int, callback func(hash string) error) erro
 				return err
 			}
 			if notFound {
-				_, err = db.deleteHitByHashStmt.Exec(hash)
+				err = db.DeleteHitAsync(hash)
 				if err != nil {
 					return db.WrapError(err)
 				}
@@ -291,13 +337,13 @@ func (this *FileList) Count() (int64, error) {
 
 // IncreaseHit 增加点击量
 func (this *FileList) IncreaseHit(hash string) error {
-	var db = this.getDB(hash)
+	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
 		return nil
 	}
 
-	return db.IncreaseHit(hash)
+	return db.IncreaseHitAsync(hash)
 }
 
 // OnAdd 添加事件
@@ -326,17 +372,23 @@ func (this *FileList) GetDBIndex(hash string) uint64 {
 	return fnv.HashString(hash) % CountFileDB
 }
 
-func (this *FileList) getDB(hash string) *FileListDB {
+func (this *FileList) GetDB(hash string) *FileListDB {
 	return this.dbList[fnv.HashString(hash)%CountFileDB]
 }
 
 func (this *FileList) remove(hash string) (notFound bool, err error) {
-	var db = this.getDB(hash)
+	var db = this.GetDB(hash)
 
 	if !db.IsReady() {
 		return false, nil
 	}
 
+	// HashMap中不存在，则确定不存在
+	if !db.hashMap.Exist(hash) {
+		return true, nil
+	}
+	defer db.hashMap.Delete(hash)
+
 	// 从缓存中删除
 	this.memoryCache.Delete(hash)
 
@@ -364,7 +416,7 @@ func (this *FileList) remove(hash string) (notFound bool, err error) {
 
 	atomic.AddInt64(&this.total, -1)
 
-	_, err = db.deleteHitByHashStmt.Exec(hash)
+	err = db.DeleteHitAsync(hash)
 	if err != nil {
 		return false, db.WrapError(err)
 	}

internal/caches/list_file_db.go

@@ -10,8 +10,13 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
+	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -25,6 +30,8 @@ type FileListDB struct {
 
 	writeBatch *dbs.Batch
 
+	hashMap *FileListHashMap
+
 	itemsTableName string
 	hitsTableName  string
 
@@ -41,47 +48,70 @@ type FileListDB struct {
 
 	selectByHashStmt *dbs.Stmt // 使用hash查询数据
 
+	selectHashListStmt *dbs.Stmt
+
 	deleteByHashStmt *dbs.Stmt // 根据hash删除数据
 	deleteByHashSQL  string
 
-	statStmt           *dbs.Stmt // 统计
-	purgeStmt          *dbs.Stmt // 清理
-	deleteAllStmt      *dbs.Stmt // 删除所有数据
-	listOlderItemsStmt *dbs.Stmt // 读取较早存储的缓存
+	statStmt            *dbs.Stmt // 统计
+	purgeStmt           *dbs.Stmt // 清理
+	deleteAllStmt       *dbs.Stmt // 删除所有数据
+	listOlderItemsStmt  *dbs.Stmt // 读取较早存储的缓存
+	updateAccessWeekSQL string    // 修改访问日期
 
 	// hits
-	insertHitStmt       *dbs.Stmt // 写入数据
-	increaseHitStmt     *dbs.Stmt // 增加点击量
-	deleteHitByHashStmt *dbs.Stmt // 根据hash删除数据
-	lfuHitsStmt         *dbs.Stmt // 读取老的数据
+	insertHitSQL       string // 写入数据
+	increaseHitSQL     string // 增加点击量
+	deleteHitByHashSQL string // 根据hash删除数据
 }
 
 func NewFileListDB() *FileListDB {
-	return &FileListDB{}
+	return &FileListDB{
+		hashMap: NewFileListHashMap(),
+	}
 }
 
 func (this *FileListDB) Open(dbPath string) error {
 	this.dbPath = dbPath
 
+	// 动态调整Cache值
+	var cacheSize = 32000
+	var memoryGB = utils.SystemMemoryGB()
+	if memoryGB >= 8 {
+		cacheSize += 32000 * memoryGB / 8
+	}
+
 	// write db
-	writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size=32000&_secure_delete=FAST")
+	writeDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=rwc&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize)+"&_secure_delete=FAST")
 	if err != nil {
 		return errors.New("open write database failed: " + err.Error())
 	}
 
 	writeDB.SetMaxOpenConns(1)
 
+	this.writeDB = dbs.NewDB(writeDB)
+
 	// TODO 耗时过长，暂时不整理数据库
 	// TODO 需要根据行数来判断是否VACUUM
+	// TODO 注意VACUUM反而可能让数据库文件变大
 	/**_, err = db.Exec("VACUUM")
 	if err != nil {
 		return err
 	}**/
 
-	this.writeDB = dbs.NewDB(writeDB)
+	// 检查是否损坏
+	// TODO 暂时屏蔽，因为用时过长
+
+	var recoverEnv, _ = os.LookupEnv("EdgeRecover")
+	if len(recoverEnv) > 0 && this.shouldRecover() {
+		for _, indexName := range []string{"staleAt", "hash"} {
+			_, _ = this.writeDB.Exec(`REINDEX "` + indexName + `"`)
+		}
+	}
+
 	this.writeBatch = dbs.NewBatch(writeDB, 4)
 	this.writeBatch.OnFail(func(err error) {
-		remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error())
+		remotelogs.Warn("LIST_FILE_DB", "run batch failed: "+err.Error()+" ("+filepath.Base(this.dbPath)+")")
 	})
 
 	goman.New(func() {
@@ -94,7 +124,7 @@ func (this *FileListDB) Open(dbPath string) error {
 	}
 
 	// read db
-	readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size=32000")
+	readDB, err := sql.Open("sqlite3", "file:"+dbPath+"?cache=private&mode=ro&_journal_mode=WAL&_sync=OFF&_cache_size="+types.String(cacheSize))
 	if err != nil {
 		return errors.New("open read database failed: " + err.Error())
 	}
@@ -138,7 +168,7 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	this.insertSQL = `INSERT INTO "` + this.itemsTableName + `" ("hash", "key", "headerSize", "bodySize", "metaSize", "expiredAt", "staleAt", "host", "serverId", "createdAt", "accessWeek") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
 	this.insertStmt, err = this.writeDB.Prepare(this.insertSQL)
 	if err != nil {
 		return err
@@ -149,6 +179,11 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
+	this.selectHashListStmt, err = this.readDB.Prepare(`SELECT "id", "hash" FROM "` + this.itemsTableName + `" WHERE id>:id ORDER BY id ASC LIMIT 2000`)
+	if err != nil {
+		return err
+	}
+
 	this.deleteByHashSQL = `DELETE FROM "` + this.itemsTableName + `" WHERE "hash"=?`
 	this.deleteByHashStmt, err = this.writeDB.Prepare(this.deleteByHashSQL)
 	if err != nil {
@@ -170,27 +205,29 @@ func (this *FileListDB) Init() error {
 		return err
 	}
 
-	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "id" ASC LIMIT ?`)
-
-	this.insertHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`)
-
-	this.increaseHitStmt, err = this.writeDB.Prepare(`INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`)
+	this.listOlderItemsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.itemsTableName + `" ORDER BY "accessWeek" ASC, "id" ASC LIMIT ?`)
 	if err != nil {
 		return err
 	}
 
-	this.deleteHitByHashStmt, err = this.writeDB.Prepare(`DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`)
-	if err != nil {
-		return err
-	}
+	this.updateAccessWeekSQL = `UPDATE "` + this.itemsTableName + `" SET "accessWeek"=? WHERE "hash"=?`
 
-	this.lfuHitsStmt, err = this.readDB.Prepare(`SELECT "hash" FROM "` + this.hitsTableName + `" ORDER BY "week" ASC, "week1Hits"+"week2Hits" ASC LIMIT ?`)
-	if err != nil {
-		return err
-	}
+	this.insertHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?)`
+
+	this.increaseHitSQL = `INSERT INTO "` + this.hitsTableName + `" ("hash", "week2Hits", "week") VALUES (?, 1, ?) ON CONFLICT("hash") DO UPDATE SET "week1Hits"=IIF("week"=?, "week1Hits", "week2Hits"), "week2Hits"=IIF("week"=?, "week2Hits"+1, 1), "week"=?`
+
+	this.deleteHitByHashSQL = `DELETE FROM "` + this.hitsTableName + `" WHERE "hash"=?`
 
 	this.isReady = true
 
+	// 加载HashMap
+	go func() {
+		err := this.hashMap.Load(this)
+		if err != nil {
+			remotelogs.Error("LIST_FILE_DB", "load hash map failed: "+err.Error()+"(file: "+this.dbPath+")")
+		}
+	}()
+
 	return nil
 }
 
@@ -203,21 +240,25 @@ func (this *FileListDB) Total() int64 {
 }
 
 func (this *FileListDB) AddAsync(hash string, item *Item) error {
+	this.hashMap.Add(hash)
+
 	if item.StaleAt == 0 {
 		item.StaleAt = item.ExpiredAt
 	}
 
-	this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
+	this.writeBatch.Add(this.insertSQL, hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime(), timeutil.Format("YW"))
 	return nil
 
 }
 
 func (this *FileListDB) AddSync(hash string, item *Item) error {
+	this.hashMap.Add(hash)
+
 	if item.StaleAt == 0 {
 		item.StaleAt = item.ExpiredAt
 	}
 
-	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime())
+	_, err := this.insertStmt.Exec(hash, item.Key, item.HeaderSize, item.BodySize, item.MetaSize, item.ExpiredAt, item.StaleAt, item.Host, item.ServerId, utils.UnixTime(), timeutil.Format("YW"))
 	if err != nil {
 		return this.WrapError(err)
 	}
@@ -226,11 +267,15 @@ func (this *FileListDB) AddSync(hash string, item *Item) error {
 }
 
 func (this *FileListDB) DeleteAsync(hash string) error {
+	this.hashMap.Delete(hash)
+
 	this.writeBatch.Add(this.deleteByHashSQL, hash)
 	return nil
 }
 
 func (this *FileListDB) DeleteSync(hash string) error {
+	this.hashMap.Delete(hash)
+
 	_, err := this.deleteByHashStmt.Exec(hash)
 	if err != nil {
 		return err
@@ -275,28 +320,56 @@ func (this *FileListDB) ListLFUItems(count int) (hashList []string, err error) {
 		count = 100
 	}
 
-	hashList, err = this.listLFUItems(count)
+	// 先找过期的
+	hashList, err = this.ListExpiredItems(count)
 	if err != nil {
 		return
 	}
+	var l = len(hashList)
 
-	if len(hashList) > count/2 {
-		return
+	// 从旧缓存中补充
+	if l < count {
+		oldHashList, err := this.listOlderItems(count - l)
+		if err != nil {
+			return nil, err
+		}
+		hashList = append(hashList, oldHashList...)
 	}
 
-	// 不足补齐
-	olderHashList, err := this.listOlderItems(count - len(hashList))
+	return hashList, nil
+}
+
+func (this *FileListDB) ListHashes(lastId int64) (hashList []string, maxId int64, err error) {
+	rows, err := this.selectHashListStmt.Query(lastId)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
-	hashList = append(hashList, olderHashList...)
+	var id int64
+	var hash string
+	for rows.Next() {
+		err = rows.Scan(&id, &hash)
+		if err != nil {
+			_ = rows.Close()
+			return
+		}
+		maxId = id
+		hashList = append(hashList, hash)
+	}
+
+	_ = rows.Close()
 	return
 }
 
-func (this *FileListDB) IncreaseHit(hash string) error {
+func (this *FileListDB) IncreaseHitAsync(hash string) error {
 	var week = timeutil.Format("YW")
-	_, err := this.increaseHitStmt.Exec(hash, week, week, week, week)
-	return this.WrapError(err)
+	this.writeBatch.Add(this.increaseHitSQL, hash, week, week, week, week)
+	this.writeBatch.Add(this.updateAccessWeekSQL, week, hash)
+	return nil
+}
+
+func (this *FileListDB) DeleteHitAsync(hash string) error {
+	this.writeBatch.Add(this.deleteHitByHashSQL, hash)
+	return nil
 }
 
 func (this *FileListDB) CleanPrefix(prefix string) error {
@@ -321,6 +394,85 @@ func (this *FileListDB) CleanPrefix(prefix string) error {
 	}
 }
 
+func (this *FileListDB) CleanMatchKey(key string) error {
+	if !this.isReady {
+		return nil
+	}
+
+	// 忽略 @GOEDGE_
+	if strings.Contains(key, SuffixAll) {
+		return nil
+	}
+
+	u, err := url.Parse(key)
+	if err != nil {
+		return nil
+	}
+
+	var host = u.Host
+	hostPart, _, err := net.SplitHostPort(host)
+	if err == nil && len(hostPart) > 0 {
+		host = hostPart
+	}
+	if len(host) == 0 {
+		return nil
+	}
+
+	// 转义
+	var queryKey = strings.ReplaceAll(key, "%", "\\%")
+	queryKey = strings.ReplaceAll(queryKey, "_", "\\_")
+	queryKey = strings.Replace(queryKey, "*", "%", 1)
+
+	// TODO 检查大批量数据下的操作性能
+	var staleLife = 600             // TODO 需要可以设置
+	var unixTime = utils.UnixTime() // 只删除当前的，不删除新的
+
+	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryKey+SuffixAll+"%")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (this *FileListDB) CleanMatchPrefix(prefix string) error {
+	if !this.isReady {
+		return nil
+	}
+
+	u, err := url.Parse(prefix)
+	if err != nil {
+		return nil
+	}
+
+	var host = u.Host
+	hostPart, _, err := net.SplitHostPort(host)
+	if err == nil && len(hostPart) > 0 {
+		host = hostPart
+	}
+	if len(host) == 0 {
+		return nil
+	}
+
+	// 转义
+	var queryPrefix = strings.ReplaceAll(prefix, "%", "\\%")
+	queryPrefix = strings.ReplaceAll(queryPrefix, "_", "\\_")
+	queryPrefix = strings.Replace(queryPrefix, "*", "%", 1)
+	queryPrefix += "%"
+
+	// TODO 检查大批量数据下的操作性能
+	var staleLife = 600             // TODO 需要可以设置
+	var unixTime = utils.UnixTime() // 只删除当前的，不删除新的
+
+	_, err = this.writeDB.Exec(`UPDATE "`+this.itemsTableName+`" SET "expiredAt"=0, "staleAt"=? WHERE "host" GLOB ? AND "host" NOT GLOB ? AND "key" LIKE ? ESCAPE '\'`, unixTime+int64(staleLife), host, "*."+host, queryPrefix)
+	return err
+}
+
 func (this *FileListDB) CleanAll() error {
 	if !this.isReady {
 		return nil
@@ -331,6 +483,8 @@ func (this *FileListDB) CleanAll() error {
 		return this.WrapError(err)
 	}
 
+	this.hashMap.Clean()
+
 	return nil
 }
 
@@ -347,6 +501,9 @@ func (this *FileListDB) Close() error {
 	if this.selectByHashStmt != nil {
 		_ = this.selectByHashStmt.Close()
 	}
+	if this.selectHashListStmt != nil {
+		_ = this.selectHashListStmt.Close()
+	}
 	if this.deleteByHashStmt != nil {
 		_ = this.deleteByHashStmt.Close()
 	}
@@ -363,17 +520,8 @@ func (this *FileListDB) Close() error {
 		_ = this.listOlderItemsStmt.Close()
 	}
 
-	if this.insertHitStmt != nil {
-		_ = this.insertHitStmt.Close()
-	}
-	if this.increaseHitStmt != nil {
-		_ = this.increaseHitStmt.Close()
-	}
-	if this.deleteHitByHashStmt != nil {
-		_ = this.deleteHitByHashStmt.Close()
-	}
-	if this.lfuHitsStmt != nil {
-		_ = this.lfuHitsStmt.Close()
+	if this.writeBatch != nil {
+		this.writeBatch.Close()
 	}
 
 	var errStrings []string
@@ -392,11 +540,6 @@ func (this *FileListDB) Close() error {
 		}
 	}
 
-
-	if this.writeBatch != nil {
-		this.writeBatch.Close()
-	}
-
 	if len(errStrings) == 0 {
 		return nil
 	}
@@ -415,6 +558,7 @@ func (this *FileListDB) initTables(times int) error {
 	{
 		// expiredAt - 过期时间，用来判断有无过期
 		// staleAt - 过时缓存最大时间，用来清理缓存
+		// 不对 hash 增加 unique 参数，是尽可能避免产生 malformed 错误
 		_, err := this.writeDB.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemsTableName + `" (
   "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
   "hash" varchar(32),
@@ -427,7 +571,8 @@ func (this *FileListDB) initTables(times int) error {
   "staleAt" integer DEFAULT 0,
   "createdAt" integer DEFAULT 0,
   "host" varchar(128),
-  "serverId" integer
+  "serverId" integer,
+  "accessWeek" varchar(6)
 );
 
 DROP INDEX IF EXISTS "createdAt";
@@ -439,23 +584,32 @@ ON "` + this.itemsTableName + `" (
   "staleAt" ASC
 );
 
-CREATE UNIQUE INDEX IF NOT EXISTS "hash"
+CREATE INDEX IF NOT EXISTS "hash"
 ON "` + this.itemsTableName + `" (
   "hash" ASC
 );
+
+ALTER TABLE "cacheItems" ADD "accessWeek" varchar(6);
 `)
 
 		if err != nil {
-			// 尝试删除重建
-			if times < 3 {
-				_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
-				if dropErr == nil {
-					return this.initTables(times + 1)
-				}
-				return this.WrapError(err)
+			// 忽略可以预期的错误
+			if strings.Contains(err.Error(), "duplicate column name") {
+				err = nil
 			}
 
-			return this.WrapError(err)
+			// 尝试删除重建
+			if err != nil {
+				if times < 3 {
+					_, dropErr := this.writeDB.Exec(`DROP TABLE "` + this.itemsTableName + `"`)
+					if dropErr == nil {
+						return this.initTables(times + 1)
+					}
+					return this.WrapError(err)
+				}
+
+				return this.WrapError(err)
+			}
 		}
 	}
 
@@ -490,27 +644,6 @@ ON "` + this.hitsTableName + `" (
 	return nil
 }
 
-func (this *FileListDB) listLFUItems(count int) (hashList []string, err error) {
-	rows, err := this.lfuHitsStmt.Query(count)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		_ = rows.Close()
-	}()
-
-	for rows.Next() {
-		var hash string
-		err = rows.Scan(&hash)
-		if err != nil {
-			return nil, err
-		}
-		hashList = append(hashList, hash)
-	}
-
-	return hashList, nil
-}
-
 func (this *FileListDB) listOlderItems(count int) (hashList []string, err error) {
 	rows, err := this.listOlderItemsStmt.Query(count)
 	if err != nil {
@@ -531,3 +664,21 @@ func (this *FileListDB) listOlderItems(count int) (hashList []string, err error)
 
 	return hashList, nil
 }
+
+func (this *FileListDB) shouldRecover() bool {
+	result, err := this.writeDB.Query("pragma integrity_check;")
+	if err != nil {
+		logs.Println(result)
+	}
+	var errString = ""
+	var shouldRecover = false
+	for result.Next() {
+		err = result.Scan(&errString)
+		if strings.TrimSpace(errString) != "ok" {
+			shouldRecover = true
+		}
+		break
+	}
+	_ = result.Close()
+	return shouldRecover
+}

internal/caches/list_file_db_test.go

@@ -0,0 +1,87 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/iwind/TeaGo/Tea"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"testing"
+	"time"
+)
+
+func TestFileListDB_ListLFUItems(t *testing.T) {
+	var db = caches.NewFileListDB()
+	err := db.Open(Tea.Root + "/data/cache-db-large.db")
+	//err := db.Open(Tea.Root + "/data/cache-index/p1/db-0.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		_ = db.Close()
+	}()
+
+	hashList, err := db.ListLFUItems(100)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("[", len(hashList), "]", hashList)
+}
+
+func TestFileListDB_IncreaseHitAsync(t *testing.T) {
+	var db = caches.NewFileListDB()
+	err := db.Open(Tea.Root + "/data/cache-db-large.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Init()
+	err = db.IncreaseHitAsync("4598e5231ba47d6ec7aa9ea640ff2eaf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// wait transaction
+	time.Sleep(1 * time.Second)
+}
+
+func TestFileListDB_CleanMatchKey(t *testing.T) {
+	var db = caches.NewFileListDB()
+	err := db.Open(Tea.Root + "/data/cache-db-large.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Init()
+
+	err = db.CleanMatchKey("https://*.goedge.cn/large-text")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.CleanMatchKey("https://*.goedge.cn:1234/large-text?%2B____")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFileListDB_CleanMatchPrefix(t *testing.T) {
+	var db = caches.NewFileListDB()
+	err := db.Open(Tea.Root + "/data/cache-db-large.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Init()
+
+	err = db.CleanMatchPrefix("https://*.goedge.cn/large-text")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = db.CleanMatchPrefix("https://*.goedge.cn:1234/large-text?%2B____")
+	if err != nil {
+		t.Fatal(err)
+	}
+}

internal/caches/list_file_hash_map.go

@@ -0,0 +1,120 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/TeaOSLab/EdgeNode/internal/zero"
+	"math/big"
+	"sync"
+)
+
+// FileListHashMap 文件Hash列表
+type FileListHashMap struct {
+	m map[uint64]zero.Zero
+
+	locker      sync.RWMutex
+	isAvailable bool
+	isReady     bool
+}
+
+func NewFileListHashMap() *FileListHashMap {
+	return &FileListHashMap{
+		m:           map[uint64]zero.Zero{},
+		isAvailable: false,
+		isReady:     false,
+	}
+}
+
+func (this *FileListHashMap) Load(db *FileListDB) error {
+	// 如果系统内存过小，我们不缓存
+	if utils.SystemMemoryGB() < 3 {
+		return nil
+	}
+
+	this.isAvailable = true
+
+	var lastId int64
+	for {
+		hashList, maxId, err := db.ListHashes(lastId)
+		if err != nil {
+			return err
+		}
+		if len(hashList) == 0 {
+			break
+		}
+		this.AddHashes(hashList)
+		lastId = maxId
+	}
+
+	this.isReady = true
+	return nil
+}
+
+func (this *FileListHashMap) Add(hash string) {
+	if !this.isAvailable {
+		return
+	}
+
+	this.locker.Lock()
+	this.m[this.bigInt(hash)] = zero.New()
+	this.locker.Unlock()
+}
+
+func (this *FileListHashMap) AddHashes(hashes []string) {
+	if !this.isAvailable {
+		return
+	}
+
+	this.locker.Lock()
+	for _, hash := range hashes {
+		this.m[this.bigInt(hash)] = zero.New()
+	}
+	this.locker.Unlock()
+}
+
+func (this *FileListHashMap) Delete(hash string) {
+	if !this.isAvailable {
+		return
+	}
+
+	this.locker.Lock()
+	delete(this.m, this.bigInt(hash))
+	this.locker.Unlock()
+}
+
+func (this *FileListHashMap) Exist(hash string) bool {
+	if !this.isAvailable {
+		return true
+	}
+	if !this.isReady {
+		// 只有完全Ready时才能判断是否为false
+		return true
+	}
+	this.locker.RLock()
+	_, ok := this.m[this.bigInt(hash)]
+	this.locker.RUnlock()
+	return ok
+}
+
+func (this *FileListHashMap) Clean() {
+	this.locker.Lock()
+	this.m = map[uint64]zero.Zero{}
+	this.locker.Unlock()
+}
+
+func (this *FileListHashMap) IsReady() bool {
+	return this.isReady
+}
+
+func (this *FileListHashMap) Len() int {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+	return len(this.m)
+}
+
+func (this *FileListHashMap) bigInt(hash string) uint64 {
+	var bigInt = big.NewInt(0)
+	bigInt.SetString(hash, 16)
+	return bigInt.Uint64()
+}

internal/caches/list_file_hash_map_test.go

@@ -0,0 +1,96 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/zero"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/types"
+	stringutil "github.com/iwind/TeaGo/utils/string"
+	"math/big"
+	"runtime"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestFileListHashMap_Memory(t *testing.T) {
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
+	var m = caches.NewFileListHashMap()
+
+	for i := 0; i < 1_000_000; i++ {
+		m.Add(stringutil.Md5(types.String(i)))
+	}
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+
+	t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
+}
+
+func TestFileListHashMap_Memory2(t *testing.T) {
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
+	var m = map[uint64]zero.Zero{}
+
+	for i := 0; i < 1_000_000; i++ {
+		m[uint64(i)] = zero.New()
+	}
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+
+	t.Log("ready", (stat2.Alloc-stat1.Alloc)/1024/1024, "M")
+}
+
+func TestFileListHashMap_BigInt(t *testing.T) {
+	for _, s := range []string{"1", "2", "3", "123", "123456"} {
+		var hash = stringutil.Md5(s)
+
+		var bigInt = big.NewInt(0)
+		bigInt.SetString(hash, 16)
+		t.Log(s, "=>", bigInt.Uint64(), "hash:", hash, "format:", strconv.FormatUint(bigInt.Uint64(), 16))
+	}
+}
+
+func TestFileListHashMap_Load(t *testing.T) {
+	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1").(*caches.FileList)
+	err := list.Init()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		_ = list.Close()
+	}()
+
+	var m = caches.NewFileListHashMap()
+	var before = time.Now()
+	var db = list.GetDB("abc")
+	err = m.Load(db)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+	t.Log("count:", m.Len())
+	m.Add("abc")
+
+	for _, hash := range []string{"33347bb4441265405347816cad36a0f8", "a", "abc", "123"} {
+		t.Log(hash, "=>", m.Exist(hash))
+	}
+}
+
+func Benchmark_BigInt(b *testing.B) {
+	var hash = stringutil.Md5("123456")
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		var bigInt = big.NewInt(0)
+		bigInt.SetString(hash, 16)
+		_ = bigInt.Uint64()
+	}
+}

internal/caches/list_file_test.go

@@ -361,19 +361,6 @@ func TestFileList_UpgradeV3(t *testing.T) {
 	t.Log("ok")
 }
 
-func TestFileList_HashList(t *testing.T) {
-	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
-	err := list.Init()
-	if err != nil {
-		t.Fatal(err)
-	}
-	prefixes, err := list.(*caches.FileList).FindAllPrefixes()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(len(prefixes))
-}
-
 func BenchmarkFileList_Exist(b *testing.B) {
 	var list = caches.NewFileList(Tea.Root + "/data/cache-index/p1")
 	err := list.Init()

internal/caches/list_interface.go

@@ -18,6 +18,12 @@ type ListInterface interface {
 	// CleanPrefix 清除某个前缀的缓存
 	CleanPrefix(prefix string) error
 
+	// CleanMatchKey 清除通配符匹配的Key
+	CleanMatchKey(key string) error
+
+	// CleanMatchPrefix 清除通配符匹配的前缀
+	CleanMatchPrefix(prefix string) error
+
 	// Remove 删除内容
 	Remove(hash string) error
 

internal/caches/list_memory.go

@@ -1,8 +1,11 @@
 package caches
 
 import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/logs"
+	"net"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
@@ -146,6 +149,82 @@ func (this *MemoryList) CleanPrefix(prefix string) error {
 	return nil
 }
 
+// CleanMatchKey 清理通配符匹配的缓存数据，类似于 https://*.example.com/hello
+func (this *MemoryList) CleanMatchKey(key string) error {
+	if strings.Contains(key, SuffixAll) {
+		return nil
+	}
+
+	u, err := url.Parse(key)
+	if err != nil {
+		return nil
+	}
+
+	var host = u.Host
+	hostPart, _, err := net.SplitHostPort(host)
+	if err == nil && len(hostPart) > 0 {
+		host = hostPart
+	}
+
+	if len(host) == 0 {
+		return nil
+	}
+	var requestURI = u.RequestURI()
+
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+
+	// TODO 需要优化性能，支持千万级数据低于1s的处理速度
+	for _, itemMap := range this.itemMaps {
+		for _, item := range itemMap {
+			if configutils.MatchDomain(host, item.Host) {
+				var itemRequestURI = item.RequestURI()
+				if itemRequestURI == requestURI || strings.HasPrefix(itemRequestURI, requestURI+SuffixAll) {
+					item.ExpiredAt = 0
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// CleanMatchPrefix 清理通配符匹配的缓存数据，类似于 https://*.example.com/prefix/
+func (this *MemoryList) CleanMatchPrefix(prefix string) error {
+	u, err := url.Parse(prefix)
+	if err != nil {
+		return nil
+	}
+
+	var host = u.Host
+	hostPart, _, err := net.SplitHostPort(host)
+	if err == nil && len(hostPart) > 0 {
+		host = hostPart
+	}
+	if len(host) == 0 {
+		return nil
+	}
+	var requestURI = u.RequestURI()
+	var isRootPath = requestURI == "/"
+
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+
+	// TODO 需要优化性能，支持千万级数据低于1s的处理速度
+	for _, itemMap := range this.itemMaps {
+		for _, item := range itemMap {
+			if configutils.MatchDomain(host, item.Host) {
+				var itemRequestURI = item.RequestURI()
+				if isRootPath || strings.HasPrefix(itemRequestURI, requestURI) {
+					item.ExpiredAt = 0
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 func (this *MemoryList) Remove(hash string) error {
 	this.locker.Lock()
 

internal/caches/manager.go

@@ -6,7 +6,6 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/iwind/TeaGo/lists"
-	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/types"
 	"strconv"
 	"sync"
@@ -16,7 +15,7 @@ var SharedManager = NewManager()
 
 func init() {
 	events.On(events.EventQuit, func() {
-		logs.Println("CACHE", "quiting cache manager")
+		remotelogs.Println("CACHE", "quiting cache manager")
 		SharedManager.UpdatePolicies([]*serverconfigs.HTTPCachePolicy{})
 	})
 }
@@ -25,7 +24,8 @@ func init() {
 type Manager struct {
 	// 全局配置
 	MaxDiskCapacity   *shared.SizeCapacity
-	DiskDir           string
+	MainDiskDir       string
+	SubDiskDirs       []*serverconfigs.CacheDir
 	MaxMemoryCapacity *shared.SizeCapacity
 
 	policyMap  map[int64]*serverconfigs.HTTPCachePolicy // policyId => []*Policy
@@ -48,12 +48,10 @@ func (this *Manager) UpdatePolicies(newPolicies []*serverconfigs.HTTPCachePolicy
 	this.locker.Lock()
 	defer this.locker.Unlock()
 
-	newPolicyIds := []int64{}
+	var newPolicyIds = []int64{}
 	for _, policy := range newPolicies {
 		// 使用节点单独的缓存目录
-		if len(this.DiskDir) > 0 {
-			policy.UpdateDiskDir(this.DiskDir)
-		}
+		policy.UpdateDiskDir(this.MainDiskDir, this.SubDiskDirs)
 
 		newPolicyIds = append(newPolicyIds, policy.Id)
 	}

internal/caches/open_file_cache.go

@@ -19,7 +19,7 @@ type OpenFileCache struct {
 	poolList *linkedlist.List
 	watcher  *fsnotify.Watcher
 
-	locker sync.Mutex
+	locker sync.RWMutex
 
 	maxSize int
 	count   int
@@ -54,13 +54,18 @@ func NewOpenFileCache(maxSize int) (*OpenFileCache, error) {
 }
 
 func (this *OpenFileCache) Get(filename string) *OpenFile {
-	this.locker.Lock()
-	defer this.locker.Unlock()
+	this.locker.RLock()
 	pool, ok := this.poolMap[filename]
+	this.locker.RUnlock()
 	if ok {
 		file, consumed := pool.Get()
 		if consumed {
+			this.locker.Lock()
 			this.count--
+
+			// pool如果为空，也不需要从列表中删除，避免put时需要重新创建
+
+			this.locker.Unlock()
 		}
 		return file
 	}
@@ -124,6 +129,9 @@ func (this *OpenFileCache) Close(filename string) {
 
 	pool, ok := this.poolMap[filename]
 	if ok {
+		// 设置关闭状态
+		pool.SetClosing()
+
 		delete(this.poolMap, filename)
 		this.poolList.Remove(pool.linkItem)
 		_ = this.watcher.Remove(filename)
@@ -146,6 +154,7 @@ func (this *OpenFileCache) CloseAll() {
 	this.poolMap = map[string]*OpenFilePool{}
 	this.poolList.Reset()
 	_ = this.watcher.Close()
+	this.count = 0
 	this.locker.Unlock()
 }
 

internal/caches/open_file_cache_test.go

@@ -0,0 +1,43 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"testing"
+	"time"
+)
+
+func TestNewOpenFileCache_Close(t *testing.T) {
+	cache, err := caches.NewOpenFileCache(1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache.Debug()
+	cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Get("b.txt")
+	cache.Get("d.txt")
+	cache.Close("a.txt")
+
+	time.Sleep(100 * time.Second)
+}
+
+func TestNewOpenFileCache_CloseAll(t *testing.T) {
+	cache, err := caches.NewOpenFileCache(1024)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache.Debug()
+	cache.Put("a.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("b.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Put("c.txt", caches.NewOpenFile(nil, nil, nil, 0))
+	cache.Get("b.txt")
+	cache.Get("d.txt")
+	cache.CloseAll()
+
+	time.Sleep(6 * time.Second)
+}

internal/caches/open_file_pool.go

@@ -12,6 +12,7 @@ type OpenFilePool struct {
 	linkItem *linkedlist.Item
 	filename string
 	version  int64
+	isClosed bool
 }
 
 func NewOpenFilePool(filename string) *OpenFilePool {
@@ -29,26 +30,43 @@ func (this *OpenFilePool) Filename() string {
 }
 
 func (this *OpenFilePool) Get() (*OpenFile, bool) {
+	// 如果已经关闭，直接返回
+	if this.isClosed {
+		return nil, false
+	}
+
 	select {
 	case file := <-this.c:
-		err := file.SeekStart()
-		if err != nil {
-			_ = file.Close()
-			return nil, true
-		}
-		file.version = this.version
+		if file != nil {
+			err := file.SeekStart()
+			if err != nil {
+				_ = file.Close()
+				return nil, true
+			}
+			file.version = this.version
 
-		return file, true
+			return file, true
+		}
+		return nil, false
 	default:
 		return nil, false
 	}
 }
 
 func (this *OpenFilePool) Put(file *OpenFile) bool {
+	// 如果已关闭，则不接受新的文件
+	if this.isClosed {
+		_ = file.Close()
+		return false
+	}
+
+	// 检查文件版本号
 	if this.version > 0 && file.version > 0 && file.version != this.version {
 		_ = file.Close()
 		return false
 	}
+
+	// 加入Pool
 	select {
 	case this.c <- file:
 		return true
@@ -63,14 +81,18 @@ func (this *OpenFilePool) Len() int {
 	return len(this.c)
 }
 
+func (this *OpenFilePool) SetClosing() {
+	this.isClosed = true
+}
+
 func (this *OpenFilePool) Close() {
-Loop:
+	this.isClosed = true
 	for {
 		select {
 		case file := <-this.c:
 			_ = file.Close()
 		default:
-			break Loop
+			return
 		}
 	}
 }

internal/caches/open_file_pool_test.go

@@ -4,6 +4,8 @@ package caches_test
 
 import (
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/iwind/TeaGo/rands"
+	"sync"
 	"testing"
 )
 
@@ -15,3 +17,30 @@ func TestOpenFilePool_Get(t *testing.T) {
 	t.Log(pool.Get())
 	t.Log(pool.Get())
 }
+
+func TestOpenFilePool_Close(t *testing.T) {
+	var pool = caches.NewOpenFilePool("a")
+	pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
+	pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
+	pool.Close()
+}
+
+func TestOpenFilePool_Concurrent(t *testing.T) {
+	var pool = caches.NewOpenFilePool("a")
+	var concurrent = 1000
+	var wg = &sync.WaitGroup{}
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func() {
+			defer wg.Done()
+
+			if rands.Int(0, 1) == 1 {
+				pool.Put(caches.NewOpenFile(nil, nil, nil, 0))
+			}
+			if rands.Int(0, 1) == 0 {
+				pool.Get()
+			}
+		}()
+	}
+	wg.Wait()
+}

internal/caches/partial_ranges.go

@@ -3,38 +3,88 @@
 package caches
 
 import (
+	"bytes"
 	"encoding/json"
-	"errors"
+	"github.com/iwind/TeaGo/types"
 	"os"
+	"strconv"
 )
 
 // PartialRanges 内容分区范围定义
 type PartialRanges struct {
-	Ranges [][2]int64 `json:"ranges"`
+	Version  int        `json:"version"`  // 版本号
+	Ranges   [][2]int64 `json:"ranges"`   // 范围
+	BodySize int64      `json:"bodySize"` // 总长度
 }
 
 // NewPartialRanges 获取新对象
-func NewPartialRanges() *PartialRanges {
-	return &PartialRanges{Ranges: [][2]int64{}}
+func NewPartialRanges(expiresAt int64) *PartialRanges {
+	return &PartialRanges{
+		Ranges:  [][2]int64{},
+		Version: 1,
+	}
+}
+
+// NewPartialRangesFromData 从数据中解析范围
+func NewPartialRangesFromData(data []byte) (*PartialRanges, error) {
+	var rs = NewPartialRanges(0)
+	for {
+		var index = bytes.IndexRune(data, '\n')
+		if index < 0 {
+			break
+		}
+		var line = data[:index]
+		var colonIndex = bytes.IndexRune(line, ':')
+		if colonIndex > 0 {
+			switch string(line[:colonIndex]) {
+			case "v": // 版本号
+				rs.Version = types.Int(line[colonIndex+1:])
+			case "b": // 总长度
+				rs.BodySize = types.Int64(line[colonIndex+1:])
+			case "r": // 范围信息
+				var commaIndex = bytes.IndexRune(line, ',')
+				if commaIndex > 0 {
+					rs.Ranges = append(rs.Ranges, [2]int64{types.Int64(line[colonIndex+1 : commaIndex]), types.Int64(line[commaIndex+1:])})
+				}
+			}
+		}
+		data = data[index+1:]
+		if len(data) == 0 {
+			break
+		}
+	}
+	return rs, nil
 }
 
 // NewPartialRangesFromJSON 从JSON中解析范围
 func NewPartialRangesFromJSON(data []byte) (*PartialRanges, error) {
-	var rs = NewPartialRanges()
+	var rs = NewPartialRanges(0)
 	err := json.Unmarshal(data, &rs)
 	if err != nil {
 		return nil, err
 	}
+	rs.Version = 0
 
 	return rs, nil
 }
 
+// NewPartialRangesFromFile 从文件中加载范围信息
 func NewPartialRangesFromFile(path string) (*PartialRanges, error) {
 	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
-	return NewPartialRangesFromJSON(data)
+	if len(data) == 0 {
+		return NewPartialRanges(0), nil
+	}
+
+	// 兼容老的JSON格式
+	if data[0] == '{' {
+		return NewPartialRangesFromJSON(data)
+	}
+
+	// 新的格式
+	return NewPartialRangesFromData(data)
 }
 
 // Add 添加新范围
@@ -105,29 +155,27 @@ func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool)
 	return
 }
 
-// AsJSON 转换为JSON
-func (this *PartialRanges) AsJSON() ([]byte, error) {
-	return json.Marshal(this)
+// 转换为字符串
+func (this *PartialRanges) String() string {
+	var s = "v:" + strconv.Itoa(this.Version) + "\n" + // version
+		"b:" + this.formatInt64(this.BodySize) + "\n" // bodySize
+	for _, r := range this.Ranges {
+		s += "r:" + this.formatInt64(r[0]) + "," + this.formatInt64(r[1]) + "\n" // range
+	}
+	return s
+}
+
+// Bytes 将内容转换为字节
+func (this *PartialRanges) Bytes() []byte {
+	return []byte(this.String())
 }
 
 // WriteToFile 写入到文件中
 func (this *PartialRanges) WriteToFile(path string) error {
-	data, err := this.AsJSON()
-	if err != nil {
-		return errors.New("convert to json failed: " + err.Error())
-	}
-	return os.WriteFile(path, data, 0666)
-}
-
-// ReadFromFile 从文件中读取
-func (this *PartialRanges) ReadFromFile(path string) (*PartialRanges, error) {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	return NewPartialRangesFromJSON(data)
+	return os.WriteFile(path, this.Bytes(), 0666)
 }
 
+// Max 获取最大位置
 func (this *PartialRanges) Max() int64 {
 	if len(this.Ranges) > 0 {
 		return this.Ranges[len(this.Ranges)-1][1]
@@ -135,6 +183,11 @@ func (this *PartialRanges) Max() int64 {
 	return 0
 }
 
+// Reset 重置范围信息
+func (this *PartialRanges) Reset() {
+	this.Ranges = [][2]int64{}
+}
+
 func (this *PartialRanges) merge(index int) {
 	// forward
 	var lastIndex = index
@@ -187,3 +240,7 @@ func (this *PartialRanges) max(n1 int64, n2 int64) int64 {
 	}
 	return n2
 }
+
+func (this *PartialRanges) formatInt64(i int64) string {
+	return strconv.FormatInt(i, 10)
+}

internal/caches/partial_ranges_test.go

@@ -3,14 +3,16 @@
 package caches_test
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
 	"github.com/iwind/TeaGo/assert"
 	"github.com/iwind/TeaGo/logs"
 	"testing"
+	"time"
 )
 
 func TestNewPartialRanges(t *testing.T) {
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	r.Add(1, 100)
 	r.Add(50, 300)
 
@@ -28,7 +30,7 @@ func TestNewPartialRanges(t *testing.T) {
 func TestNewPartialRanges1(t *testing.T) {
 	var a = assert.NewAssertion(t)
 
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	r.Add(1, 100)
 	r.Add(1, 101)
 	r.Add(1, 102)
@@ -47,7 +49,7 @@ func TestNewPartialRanges1(t *testing.T) {
 
 func TestNewPartialRanges2(t *testing.T) {
 	// low -> high
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	r.Add(1, 100)
 	r.Add(1, 101)
 	r.Add(1, 102)
@@ -63,7 +65,7 @@ func TestNewPartialRanges2(t *testing.T) {
 
 func TestNewPartialRanges3(t *testing.T) {
 	// high -> low
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	r.Add(301, 302)
 	r.Add(303, 304)
 	r.Add(200, 300)
@@ -75,7 +77,7 @@ func TestNewPartialRanges3(t *testing.T) {
 
 func TestNewPartialRanges4(t *testing.T) {
 	// nearby
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	r.Add(301, 302)
 	r.Add(303, 304)
 	r.Add(305, 306)
@@ -90,7 +92,7 @@ func TestNewPartialRanges4(t *testing.T) {
 }
 
 func TestNewPartialRanges5(t *testing.T) {
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	for j := 0; j < 1000; j++ {
 		r.Add(int64(j), int64(j+100))
 	}
@@ -100,7 +102,7 @@ func TestNewPartialRanges5(t *testing.T) {
 func TestNewPartialRanges_Nearest(t *testing.T) {
 	{
 		// nearby
-		var r = caches.NewPartialRanges()
+		var r = caches.NewPartialRanges(0)
 		r.Add(301, 400)
 		r.Add(401, 500)
 		r.Add(501, 600)
@@ -112,7 +114,7 @@ func TestNewPartialRanges_Nearest(t *testing.T) {
 
 	{
 		// nearby
-		var r = caches.NewPartialRanges()
+		var r = caches.NewPartialRanges(0)
 		r.Add(301, 400)
 		r.Add(450, 500)
 		r.Add(550, 600)
@@ -131,45 +133,100 @@ func TestNewPartialRanges_Large_Range(t *testing.T) {
 	var largeSize int64 = 10000000000000
 	t.Log(largeSize/1024/1024/1024, "G")
 
-	var r = caches.NewPartialRanges()
+	var r = caches.NewPartialRanges(0)
 	r.Add(1, largeSize)
-	jsonData, err := r.AsJSON()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(string(jsonData))
+	var s = r.String()
+	t.Log(s)
 
-	r2, err := caches.NewPartialRangesFromJSON(jsonData)
+	r2, err := caches.NewPartialRangesFromData([]byte(s))
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	a.IsTrue(largeSize == r2.Ranges[0][1])
+	logs.PrintAsJSON(r, t)
 }
 
-func TestNewPartialRanges_AsJSON(t *testing.T) {
-	var r = caches.NewPartialRanges()
-	for j := 0; j < 1000; j++ {
-		r.Add(int64(j), int64(j+100))
+func TestPartialRanges_Encode_JSON(t *testing.T) {
+	var r = caches.NewPartialRanges(0)
+	for i := 0; i < 10; i++ {
+		r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
 	}
-	data, err := r.AsJSON()
+	var before = time.Now()
+	data, err := json.Marshal(r)
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Log(string(data))
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+	t.Log(len(data))
+}
 
-	r2, err := caches.NewPartialRangesFromJSON(data)
+func TestPartialRanges_Encode_String(t *testing.T) {
+	var r = caches.NewPartialRanges(0)
+	r.BodySize = 1024
+	for i := 0; i < 10; i++ {
+		r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
+	}
+	var before = time.Now()
+	var data = r.String()
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+	t.Log(len(data))
+
+	r2, err := caches.NewPartialRangesFromData([]byte(data))
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Log(r2.Ranges)
+	logs.PrintAsJSON(r2, t)
+}
+
+func TestPartialRanges_Version(t *testing.T) {
+	{
+		ranges, err := caches.NewPartialRangesFromData([]byte(`e:1668928495
+r:0,1048576
+r:1140260864,1140295164`))
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("version:", ranges.Version)
+	}
+	{
+		ranges, err := caches.NewPartialRangesFromData([]byte(`e:1668928495
+r:0,1048576
+r:1140260864,1140295164
+v:0
+`))
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("version:", ranges.Version)
+	}
+	{
+		ranges, err := caches.NewPartialRangesFromJSON([]byte(`{}`))
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("version:", ranges.Version)
+	}
 }
 
 func BenchmarkNewPartialRanges(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		var r = caches.NewPartialRanges()
+		var r = caches.NewPartialRanges(0)
 		for j := 0; j < 1000; j++ {
 			r.Add(int64(j), int64(j+100))
 		}
 	}
 }
+
+func BenchmarkPartialRanges_String(b *testing.B) {
+	var r = caches.NewPartialRanges(0)
+	r.BodySize = 1024
+	for i := 0; i < 10; i++ {
+		r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
+	}
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_ = r.String()
+	}
+}

internal/caches/reader_file.go

@@ -42,7 +42,7 @@ func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
 		this.header = this.openFile.header
 	}
 
-	isOk := false
+	var isOk = false
 
 	if autoDiscard {
 		defer func() {
@@ -67,17 +67,17 @@ func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
 
 	this.expiresAt = int64(binary.BigEndian.Uint32(buf[:SizeExpiresAt]))
 
-	status := types.Int(string(buf[OffsetStatus : OffsetStatus+SizeStatus]))
+	var status = types.Int(string(buf[OffsetStatus : OffsetStatus+SizeStatus]))
 	if status < 100 || status > 999 {
 		return errors.New("invalid status")
 	}
 	this.status = status
 
 	// URL
-	urlLength := binary.BigEndian.Uint32(buf[OffsetURLLength : OffsetURLLength+SizeURLLength])
+	var urlLength = binary.BigEndian.Uint32(buf[OffsetURLLength : OffsetURLLength+SizeURLLength])
 
 	// header
-	headerSize := int(binary.BigEndian.Uint32(buf[OffsetHeaderLength : OffsetHeaderLength+SizeHeaderLength]))
+	var headerSize = int(binary.BigEndian.Uint32(buf[OffsetHeaderLength : OffsetHeaderLength+SizeHeaderLength]))
 	if headerSize == 0 {
 		return nil
 	}
@@ -86,7 +86,7 @@ func (this *FileReader) InitAutoDiscard(autoDiscard bool) error {
 
 	// body
 	this.bodyOffset = this.headerOffset + int64(headerSize)
-	bodySize := int(binary.BigEndian.Uint64(buf[OffsetBodyLength : OffsetBodyLength+SizeBodyLength]))
+	var bodySize = int(binary.BigEndian.Uint64(buf[OffsetBodyLength : OffsetBodyLength+SizeBodyLength]))
 	if bodySize == 0 {
 		isOk = true
 		return nil
@@ -158,7 +158,7 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
 		return nil
 	}
 
-	isOk := false
+	var isOk = false
 
 	defer func() {
 		if !isOk {
@@ -171,7 +171,7 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
 		return err
 	}
 
-	headerSize := this.headerSize
+	var headerSize = this.headerSize
 
 	for {
 		n, err := this.fp.Read(buf)
@@ -215,7 +215,11 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
 }
 
 func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
-	isOk := false
+	if this.bodySize == 0 {
+		return nil
+	}
+
+	var isOk = false
 
 	defer func() {
 		if !isOk {
@@ -257,15 +261,22 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
 }
 
 func (this *FileReader) Read(buf []byte) (n int, err error) {
+	if this.bodySize == 0 {
+		n = 0
+		err = io.EOF
+		return
+	}
+
 	n, err = this.fp.Read(buf)
 	if err != nil && err != io.EOF {
 		_ = this.discard()
 	}
+
 	return
 }
 
 func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error {
-	isOk := false
+	var isOk = false
 
 	defer func() {
 		if !isOk {
@@ -273,7 +284,7 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
 		}
 	}()
 
-	offset := start
+	var offset = start
 	if start < 0 {
 		offset = this.bodyOffset + this.bodySize + end
 		end = this.bodyOffset + this.bodySize - 1
@@ -296,7 +307,7 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
 	for {
 		n, err := this.fp.Read(buf)
 		if n > 0 {
-			n2 := int(end-offset) + 1
+			var n2 = int(end-offset) + 1
 			if n2 <= n {
 				_, e := callback(n2)
 				if e != nil {
@@ -344,12 +355,12 @@ func (this *FileReader) FP() *os.File {
 }
 
 func (this *FileReader) Close() error {
-	if this.openFileCache != nil {
-		if this.isClosed {
-			return nil
-		}
-		this.isClosed = true
+	if this.isClosed {
+		return nil
+	}
+	this.isClosed = true
 
+	if this.openFileCache != nil {
 		if this.openFile != nil {
 			this.openFileCache.Put(this.fp.Name(), this.openFile)
 		} else {
@@ -359,6 +370,7 @@ func (this *FileReader) Close() error {
 		}
 		return nil
 	}
+
 	return this.fp.Close()
 }
 

internal/caches/reader_file_test.go

@@ -19,7 +19,7 @@ func TestFileReader(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, path := storage.keyPath("my-key")
+	_, path, _ := storage.keyPath("my-key")
 
 	fp, err := os.Open(path)
 	if err != nil {
@@ -105,7 +105,7 @@ func TestFileReader_Range(t *testing.T) {
 	}
 	_ = writer.Close()**/
 
-	_, path := storage.keyPath("my-number")
+	_, path, _ := storage.keyPath("my-number")
 
 	fp, err := os.Open(path)
 	if err != nil {

internal/caches/reader_partial_file.go

@@ -117,13 +117,10 @@ func (this *PartialFileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.
 	r2, ok = this.ranges.Nearest(r.Start(), r.End())
 	if ok && this.bodySize > 0 {
 		// 考虑可配置
-		var span int64 = 512 * 1024
-		if this.bodySize > 1<<30 {
-			span = 1 << 20
-		}
+		const minSpan = 128 << 10
 
 		// 这里限制返回的最小缓存，防止因为返回的内容过小而导致请求过多
-		if r2.Length() < r.Length() && r2.Length() < span {
+		if r2.Length() < r.Length() && r2.Length() < minSpan {
 			ok = false
 		}
 	}
@@ -138,6 +135,10 @@ func (this *PartialFileReader) MaxLength() int64 {
 	return this.ranges.Max() + 1
 }
 
+func (this *PartialFileReader) Ranges() *PartialRanges {
+	return this.ranges
+}
+
 func (this *PartialFileReader) discard() error {
 	_ = os.Remove(this.rangePath)
 	return this.FileReader.discard()

internal/caches/storage_file.go

@@ -21,6 +21,7 @@ import (
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
 	stringutil "github.com/iwind/TeaGo/utils/string"
+	"golang.org/x/sys/unix"
 	"golang.org/x/text/language"
 	"golang.org/x/text/message"
 	"math"
@@ -48,8 +49,7 @@ const (
 	SizeBodyLength     = 8
 	OffsetBodyLength   = OffsetHeaderLength + SizeHeaderLength
 
-	SizeMeta  = SizeExpiresAt + SizeStatus + SizeURLLength + SizeHeaderLength + SizeBodyLength
-	OffsetKey = SizeMeta
+	SizeMeta = SizeExpiresAt + SizeStatus + SizeURLLength + SizeHeaderLength + SizeBodyLength
 )
 
 const (
@@ -58,6 +58,7 @@ const (
 	HotItemLifeSeconds       int64 = 3600         // 热点数据生命周期
 	FileToMemoryMaxSize            = 32 * sizes.M // 可以从文件写入到内存的最大文件尺寸
 	FileTmpSuffix                  = ".tmp"
+	MinDiskSpace                   = 5 << 30 // 当前磁盘最小剩余空间
 )
 
 var sharedWritingFileKeyMap = map[string]zero.Zero{} // key => bool
@@ -90,6 +91,11 @@ type FileStorage struct {
 	ignoreKeys *setutils.FixedSet
 
 	openFileCache *OpenFileCache
+
+	mainDir        string
+	mainDiskIsFull bool
+
+	subDirs []*FileDir
 }
 
 func NewFileStorage(policy *serverconfigs.HTTPCachePolicy) *FileStorage {
@@ -153,6 +159,16 @@ func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy)
 		return
 	}
 
+	var subDirs = []*FileDir{}
+	for _, subDir := range newOptions.SubDirs {
+		subDirs = append(subDirs, &FileDir{
+			Path:     subDir.Path,
+			Capacity: subDir.Capacity,
+			IsFull:   false,
+		})
+	}
+	this.checkDiskSpace()
+
 	err = newOptions.Init()
 	if err != nil {
 		remotelogs.Error("CACHE", "update policy '"+types.String(this.policy.Id)+"' failed: init options failed: "+err.Error())
@@ -179,7 +195,7 @@ func (this *FileStorage) UpdatePolicy(newPolicy *serverconfigs.HTTPCachePolicy)
 	// open cache
 	oldOpenFileCacheJSON, _ := json.Marshal(oldOpenFileCache)
 	newOpenFileCacheJSON, _ := json.Marshal(this.options.OpenFileCache)
-	if bytes.Compare(oldOpenFileCacheJSON, newOpenFileCacheJSON) != 0 {
+	if !bytes.Equal(oldOpenFileCacheJSON, newOpenFileCacheJSON) {
 		this.initOpenFileCache()
 	}
 
@@ -215,6 +231,19 @@ func (this *FileStorage) Init() error {
 	this.options.Dir = filepath.Clean(this.options.Dir)
 	var dir = this.options.Dir
 
+	var subDirs = []*FileDir{}
+	for _, subDir := range this.options.SubDirs {
+		subDirs = append(subDirs, &FileDir{
+			Path:     subDir.Path,
+			Capacity: subDir.Capacity,
+			IsFull:   false,
+		})
+	}
+	this.subDirs = subDirs
+	if len(subDirs) > 0 {
+		this.checkDiskSpace()
+	}
+
 	if len(dir) == 0 {
 		return errors.New("[CACHE]cache storage dir can not be empty")
 	}
@@ -287,6 +316,9 @@ func (this *FileStorage) Init() error {
 	// open file cache
 	this.initOpenFileCache()
 
+	// 检查磁盘空间
+	this.checkDiskSpace()
+
 	return nil
 }
 
@@ -314,7 +346,7 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
 		}
 	}
 
-	hash, path := this.keyPath(key)
+	hash, path, _ := this.keyPath(key)
 
 	// 检查文件记录是否已过期
 	if !useStale {
@@ -382,16 +414,16 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
 }
 
 // OpenWriter 打开缓存文件等待写入
-func (this *FileStorage) OpenWriter(key string, expiresAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error) {
-	return this.openWriter(key, expiresAt, status, size, maxSize, isPartial, false)
+func (this *FileStorage) OpenWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error) {
+	return this.openWriter(key, expiresAt, status, headerSize, bodySize, maxSize, isPartial, false)
 }
 
 // OpenFlushWriter 打开从其他媒介直接刷入的写入器
-func (this *FileStorage) OpenFlushWriter(key string, expiresAt int64, status int) (Writer, error) {
-	return this.openWriter(key, expiresAt, status, -1, -1, false, true)
+func (this *FileStorage) OpenFlushWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64) (Writer, error) {
+	return this.openWriter(key, expiresAt, status, headerSize, bodySize, -1, false, true)
 }
 
-func (this *FileStorage) openWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool, isFlushing bool) (Writer, error) {
+func (this *FileStorage) openWriter(key string, expiredAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool, isFlushing bool) (Writer, error) {
 	// 是否正在退出
 	if teaconst.IsQuiting {
 		return nil, ErrWritingUnavailable
@@ -409,8 +441,8 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 		maxMemorySize = maxSize
 	}
 	var memoryStorage = this.memoryStorage
-	if !isFlushing && !isPartial && memoryStorage != nil && ((size > 0 && size < maxMemorySize) || size < 0) {
-		writer, err := memoryStorage.OpenWriter(key, expiredAt, status, size, maxMemorySize, false)
+	if !isFlushing && !isPartial && memoryStorage != nil && ((bodySize > 0 && bodySize < maxMemorySize) || bodySize < 0) {
+		writer, err := memoryStorage.OpenWriter(key, expiredAt, status, headerSize, bodySize, maxMemorySize, false)
 		if err == nil {
 			return writer, nil
 		}
@@ -463,17 +495,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 
 	var hash = stringutil.Md5(key)
 
-	// TODO 可以只stat一次
-	var dir = this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
-	_, err = os.Stat(dir)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return nil, err
-		}
-		err = os.MkdirAll(dir, 0777)
-		if err != nil {
-			return nil, err
-		}
+	dir, diskIsFull := this.subDir(hash)
+	if diskIsFull {
+		return nil, NewCapacityError("the disk is full")
 	}
 
 	// 检查缓存是否已经生成
@@ -520,19 +544,38 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 	// 从已经存储的内容中读取信息
 	var isNewCreated = true
 	var partialBodyOffset int64
+	var partialRanges *PartialRanges
 	if isPartial {
-		readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
-		if err == nil {
-			var partialReader = NewPartialFileReader(readerFp)
-			err = partialReader.Init()
-			_ = partialReader.Close()
-			if err == nil && partialReader.bodyOffset > 0 {
-				isNewCreated = false
-				partialBodyOffset = partialReader.bodyOffset
-			} else {
-				_ = this.removeCacheFile(tmpPath)
+		// 数据库中是否存在
+		existsCacheItem, _ := this.list.Exist(hash)
+		if existsCacheItem {
+			readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
+			if err == nil {
+				var partialReader = NewPartialFileReader(readerFp)
+				err = partialReader.Init()
+				_ = partialReader.Close()
+				if err == nil && partialReader.bodyOffset > 0 {
+					partialRanges = partialReader.Ranges()
+					if bodySize > 0 && partialRanges != nil && partialRanges.BodySize > 0 && bodySize != partialRanges.BodySize {
+						_ = this.removeCacheFile(tmpPath)
+					} else {
+						isNewCreated = false
+						partialBodyOffset = partialReader.bodyOffset
+					}
+				} else {
+					_ = this.removeCacheFile(tmpPath)
+				}
 			}
 		}
+		if isNewCreated {
+			err = this.list.Remove(hash)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if partialRanges == nil {
+			partialRanges = NewPartialRanges(expiredAt)
+		}
 	}
 
 	var flags = os.O_CREATE | os.O_WRONLY
@@ -542,7 +585,16 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 	var before = time.Now()
 	writer, err := os.OpenFile(tmpPath, flags, 0666)
 	if err != nil {
-		return nil, err
+		// TODO 检查在各个系统中的稳定性
+		if os.IsNotExist(err) {
+			_ = os.MkdirAll(dir, 0777)
+
+			// open file again
+			writer, err = os.OpenFile(tmpPath, flags, 0666)
+		}
+		if err != nil {
+			return nil, err
+		}
 	}
 	if !isFlushing {
 		if time.Since(before) >= maxOpenFilesSlowCost {
@@ -574,9 +626,12 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 		return nil, ErrFileIsWriting
 	}
 
+	var metaBodySize int64 = -1
+	var metaHeaderSize = -1
 	if isNewCreated {
-		// 写入过期时间
-		var metaBytes = make([]byte, SizeMeta+len(key))
+		// 写入meta
+		// 从v0.5.8开始不再在meta中写入Key
+		var metaBytes = make([]byte, SizeMeta)
 		binary.BigEndian.PutUint32(metaBytes[OffsetExpiresAt:], uint32(expiredAt))
 
 		// 写入状态码
@@ -585,17 +640,17 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 		}
 		copy(metaBytes[OffsetStatus:], strconv.Itoa(status))
 
-		// 写入URL长度
-		binary.BigEndian.PutUint32(metaBytes[OffsetURLLength:], uint32(len(key)))
-
 		// 写入Header Length
-		binary.BigEndian.PutUint32(metaBytes[OffsetHeaderLength:], uint32(0))
+		if headerSize > 0 {
+			binary.BigEndian.PutUint32(metaBytes[OffsetHeaderLength:], uint32(headerSize))
+			metaHeaderSize = headerSize
+		}
 
 		// 写入Body Length
-		binary.BigEndian.PutUint64(metaBytes[OffsetBodyLength:], uint64(0))
-
-		// 写入URL
-		copy(metaBytes[OffsetKey:], key)
+		if bodySize > 0 {
+			binary.BigEndian.PutUint64(metaBytes[OffsetBodyLength:], uint64(bodySize))
+			metaBodySize = bodySize
+		}
 
 		_, err = writer.Write(metaBytes)
 		if err != nil {
@@ -605,12 +660,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 
 	isOk = true
 	if isPartial {
-		ranges, err := NewPartialRangesFromFile(cachePathName + "@ranges.cache")
-		if err != nil {
-			ranges = NewPartialRanges()
-		}
-
-		return NewPartialFileWriter(writer, key, expiredAt, isNewCreated, isPartial, partialBodyOffset, ranges, func() {
+		return NewPartialFileWriter(writer, key, expiredAt, metaHeaderSize, metaBodySize, isNewCreated, isPartial, partialBodyOffset, partialRanges, func() {
 			sharedWritingFileKeyLocker.Lock()
 			delete(sharedWritingFileKeyMap, key)
 			if len(sharedWritingFileKeyMap) == 0 {
@@ -619,7 +669,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, siz
 			sharedWritingFileKeyLocker.Unlock()
 		}), nil
 	} else {
-		return NewFileWriter(this, writer, key, expiredAt, -1, func() {
+		return NewFileWriter(this, writer, key, expiredAt, metaHeaderSize, metaBodySize, -1, func() {
 			sharedWritingFileKeyLocker.Lock()
 			delete(sharedWritingFileKeyMap, key)
 			if len(sharedWritingFileKeyMap) == 0 {
@@ -646,7 +696,7 @@ func (this *FileStorage) AddToList(item *Item) {
 	}
 
 	item.MetaSize = SizeMeta + 128
-	hash := stringutil.Md5(item.Key)
+	var hash = stringutil.Md5(item.Key)
 	err := this.list.Add(hash, item)
 	if err != nil && !strings.Contains(err.Error(), "UNIQUE constraint failed") {
 		remotelogs.Error("CACHE", "add to list failed: "+err.Error())
@@ -660,15 +710,12 @@ func (this *FileStorage) Delete(key string) error {
 		return nil
 	}
 
-	this.locker.Lock()
-	defer this.locker.Unlock()
-
 	// 先尝试内存缓存
 	this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
 		_ = memoryStorage.Delete(key)
 	})
 
-	hash, path := this.keyPath(key)
+	hash, path, _ := this.keyPath(key)
 	err := this.list.Remove(hash)
 	if err != nil {
 		return err
@@ -683,9 +730,6 @@ func (this *FileStorage) Delete(key string) error {
 
 // Stat 统计
 func (this *FileStorage) Stat() (*Stat, error) {
-	this.locker.RLock()
-	defer this.locker.RUnlock()
-
 	return this.list.Stat(func(hash string) bool {
 		return true
 	})
@@ -708,57 +752,72 @@ func (this *FileStorage) CleanAll() error {
 
 	// 删除缓存和目录
 	// 不能直接删除子目录，比较危险
-	dir := this.dir()
-	fp, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = fp.Close()
-	}()
 
-	stat, err := fp.Stat()
-	if err != nil {
-		return err
-	}
-
-	if !stat.IsDir() {
-		return nil
-	}
-
-	// 改成待删除
-	subDirs, err := fp.Readdir(-1)
-	if err != nil {
-		return err
-	}
-	for _, info := range subDirs {
-		subDir := info.Name()
-
-		// 检查目录名
-		ok, err := regexp.MatchString(`^[0-9a-f]{2}$`, subDir)
-		if err != nil {
-			return err
-		}
-		if !ok {
-			continue
+	var rootDirs = []string{this.options.Dir}
+	var subDirs = this.subDirs // copy slice
+	if len(subDirs) > 0 {
+		for _, subDir := range subDirs {
+			rootDirs = append(rootDirs, subDir.Path)
 		}
+	}
 
-		// 修改目录名
-		tmpDir := dir + "/" + subDir + "-deleted"
-		err = os.Rename(dir+"/"+subDir, tmpDir)
+	var dirNameReg = regexp.MustCompile(`^[0-9a-f]{2}$`)
+	for _, rootDir := range rootDirs {
+		var dir = rootDir + "/p" + types.String(this.policy.Id)
+		err = func(dir string) error {
+			fp, err := os.Open(dir)
+			if err != nil {
+				return err
+			}
+			defer func() {
+				_ = fp.Close()
+			}()
+
+			stat, err := fp.Stat()
+			if err != nil {
+				return err
+			}
+
+			if !stat.IsDir() {
+				return nil
+			}
+
+			// 改成待删除
+			subDirs, err := fp.Readdir(-1)
+			if err != nil {
+				return err
+			}
+			for _, info := range subDirs {
+				subDir := info.Name()
+
+				// 检查目录名
+				if !dirNameReg.MatchString(subDir) {
+					continue
+				}
+
+				// 修改目录名
+				tmpDir := dir + "/" + subDir + "-deleted"
+				err = os.Rename(dir+"/"+subDir, tmpDir)
+				if err != nil {
+					return err
+				}
+			}
+
+			// 重新遍历待删除
+			goman.New(func() {
+				err = this.cleanDeletedDirs(dir)
+				if err != nil {
+					remotelogs.Warn("CACHE", "delete '*-deleted' dirs failed: "+err.Error())
+				}
+			})
+
+			return nil
+		}(dir)
 		if err != nil {
 			return err
 		}
 	}
 
-	// 重新遍历待删除
-	goman.New(func() {
-		err = this.cleanDeletedDirs(dir)
-		if err != nil {
-			remotelogs.Warn("CACHE", "delete '*-deleted' dirs failed: "+err.Error())
-		}
-	})
-
 	return nil
 }
 
@@ -769,9 +828,6 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
 		return nil
 	}
 
-	this.locker.Lock()
-	defer this.locker.Unlock()
-
 	// 先尝试内存缓存
 	this.runMemoryStorageSafety(func(memoryStorage *MemoryStorage) {
 		_ = memoryStorage.Purge(keys, urlType)
@@ -780,6 +836,19 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
 	// 目录
 	if urlType == "dir" {
 		for _, key := range keys {
+			// 检查是否有通配符 http(s)://*.example.com
+			var schemeIndex = strings.Index(key, "://")
+			if schemeIndex > 0 {
+				var keyRight = key[schemeIndex+3:]
+				if strings.HasPrefix(keyRight, "*.") {
+					err := this.list.CleanMatchPrefix(key)
+					if err != nil {
+						return err
+					}
+					continue
+				}
+			}
+
 			err := this.list.CleanPrefix(key)
 			if err != nil {
 				return err
@@ -790,7 +859,21 @@ func (this *FileStorage) Purge(keys []string, urlType string) error {
 
 	// URL
 	for _, key := range keys {
-		hash, path := this.keyPath(key)
+		// 检查是否有通配符 http(s)://*.example.com
+		var schemeIndex = strings.Index(key, "://")
+		if schemeIndex > 0 {
+			var keyRight = key[schemeIndex+3:]
+			if strings.HasPrefix(keyRight, "*.") {
+				err := this.list.CleanMatchKey(key)
+				if err != nil {
+					return err
+				}
+				continue
+			}
+		}
+
+		// 普通的Key
+		hash, path, _ := this.keyPath(key)
 		err := this.removeCacheFile(path)
 		if err != nil && !os.IsNotExist(err) {
 			return err
@@ -861,25 +944,22 @@ func (this *FileStorage) CanSendfile() bool {
 	return this.options.EnableSendfile
 }
 
-// 绝对路径
-func (this *FileStorage) dir() string {
-	return this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/"
-}
-
 // 获取Key对应的文件路径
-func (this *FileStorage) keyPath(key string) (hash string, path string) {
+func (this *FileStorage) keyPath(key string) (hash string, path string, diskIsFull bool) {
 	hash = stringutil.Md5(key)
-	dir := this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
+	var dir string
+	dir, diskIsFull = this.subDir(hash)
 	path = dir + "/" + hash + ".cache"
 	return
 }
 
 // 获取Hash对应的文件路径
-func (this *FileStorage) hashPath(hash string) (path string) {
+func (this *FileStorage) hashPath(hash string) (path string, diskIsFull bool) {
 	if len(hash) != 32 {
-		return ""
+		return "", false
 	}
-	dir := this.options.Dir + "/p" + strconv.FormatInt(this.policy.Id, 10) + "/" + hash[:2] + "/" + hash[2:4]
+	var dir string
+	dir, diskIsFull = this.subDir(hash)
 	path = dir + "/" + hash + ".cache"
 	return
 }
@@ -937,19 +1017,39 @@ func (this *FileStorage) initList() error {
 }
 
 // 清理任务
+// TODO purge每个分区
 func (this *FileStorage) purgeLoop() {
+	// 检查磁盘剩余空间
+	this.checkDiskSpace()
+
 	// 计算是否应该开启LFU清理
-	var capacityBytes = this.policy.CapacityBytes()
+	var capacityBytes = this.diskCapacityBytes()
 	var startLFU = false
-	var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
 	var lfuFreePercent = this.policy.PersistenceLFUFreePercent
 	if lfuFreePercent <= 0 {
 		lfuFreePercent = 5
 	}
-	if capacityBytes > 0 {
-		if lfuFreePercent < 100 {
-			if usedPercent >= 100-lfuFreePercent {
-				startLFU = true
+
+	var hasFullDisk = this.mainDiskIsFull
+	if !hasFullDisk {
+		var subDirs = this.subDirs // copy slice
+		for _, subDir := range subDirs {
+			if subDir.IsFull {
+				hasFullDisk = true
+				break
+			}
+		}
+	}
+
+	if hasFullDisk {
+		startLFU = true
+	} else {
+		var usedPercent = float32(this.TotalDiskSize()*100) / float32(capacityBytes)
+		if capacityBytes > 0 {
+			if lfuFreePercent < 100 {
+				if usedPercent >= 100-lfuFreePercent {
+					startLFU = true
+				}
 			}
 		}
 	}
@@ -974,7 +1074,7 @@ func (this *FileStorage) purgeLoop() {
 		}
 		for i := 0; i < times; i++ {
 			countFound, err := this.list.Purge(purgeCount, func(hash string) error {
-				path := this.hashPath(hash)
+				path, _ := this.hashPath(hash)
 				err := this.removeCacheFile(path)
 				if err != nil && !os.IsNotExist(err) {
 					remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
@@ -1008,7 +1108,7 @@ func (this *FileStorage) purgeLoop() {
 
 				remotelogs.Println("CACHE", "LFU purge policy '"+this.policy.Name+"' id: "+types.String(this.policy.Id)+", count: "+types.String(count))
 				err := this.list.PurgeLFU(count, func(hash string) error {
-					path := this.hashPath(hash)
+					path, _ := this.hashPath(hash)
 					err := this.removeCacheFile(path)
 					if err != nil && !os.IsNotExist(err) {
 						remotelogs.Error("CACHE", "purge '"+path+"' error: "+err.Error())
@@ -1089,7 +1189,7 @@ func (this *FileStorage) hotLoop() {
 				expiresAt = bestExpiresAt
 			}
 
-			writer, err := memoryStorage.openWriter(item.Key, expiresAt, reader.Status(), reader.BodySize(), -1, false)
+			writer, err := memoryStorage.openWriter(item.Key, expiresAt, reader.Status(), types.Int(reader.HeaderSize()), reader.BodySize(), -1, false)
 			if err != nil {
 				if !CanIgnoreErr(err) {
 					remotelogs.Error("CACHE", "transfer hot item failed: "+err.Error())
@@ -1113,9 +1213,12 @@ func (this *FileStorage) hotLoop() {
 			}
 
 			err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
-				_, err = writer.Write(buf[:n])
-				if err == nil {
-					goNext = true
+				goNext = true
+				if n > 0 {
+					_, err = writer.Write(buf[:n])
+					if err != nil {
+						goNext = false
+					}
 				}
 				return
 			})
@@ -1128,6 +1231,7 @@ func (this *FileStorage) hotLoop() {
 			memoryStorage.AddToList(&Item{
 				Type:       writer.ItemType(),
 				Key:        item.Key,
+				Host:       ParseHost(item.Key),
 				ExpiredAt:  expiresAt,
 				HeaderSize: writer.HeaderSize(),
 				BodySize:   writer.BodySize(),
@@ -1327,3 +1431,63 @@ func (this *FileStorage) runMemoryStorageSafety(f func(memoryStorage *MemoryStor
 		f(memoryStorage)
 	}
 }
+
+// 检查磁盘剩余空间
+func (this *FileStorage) checkDiskSpace() {
+	if this.options != nil && len(this.options.Dir) > 0 {
+		var stat unix.Statfs_t
+		err := unix.Statfs(this.options.Dir, &stat)
+		if err == nil {
+			var availableBytes = stat.Bavail * uint64(stat.Bsize)
+			this.mainDiskIsFull = availableBytes < MinDiskSpace
+		}
+	}
+	var subDirs = this.subDirs // copy slice
+	for _, subDir := range subDirs {
+		var stat unix.Statfs_t
+		err := unix.Statfs(subDir.Path, &stat)
+		if err == nil {
+			var availableBytes = stat.Bavail * uint64(stat.Bsize)
+			subDir.IsFull = availableBytes < MinDiskSpace
+		}
+	}
+}
+
+// 获取目录
+func (this *FileStorage) subDir(hash string) (dirPath string, dirIsFull bool) {
+	var suffix = "/p" + types.String(this.policy.Id) + "/" + hash[:2] + "/" + hash[2:4]
+
+	if len(hash) < 4 {
+		return this.options.Dir + suffix, this.mainDiskIsFull
+	}
+
+	var subDirs = this.subDirs // copy slice
+	var countSubDirs = len(subDirs)
+	if countSubDirs == 0 {
+		return this.options.Dir + suffix, this.mainDiskIsFull
+	}
+
+	countSubDirs++ // add main dir
+
+	// 最多只支持16个目录
+	if countSubDirs > 16 {
+		countSubDirs = 16
+	}
+
+	var dirIndex = this.charCode(hash[0]) % uint8(countSubDirs)
+	if dirIndex == 0 {
+		return this.options.Dir + suffix, this.mainDiskIsFull
+	}
+	var subDir = subDirs[dirIndex-1]
+	return subDir.Path + suffix, subDir.IsFull
+}
+
+func (this *FileStorage) charCode(r byte) uint8 {
+	if r >= '0' && r <= '9' {
+		return r - '0'
+	}
+	if r >= 'a' && r <= 'z' {
+		return r - 'a' + 10
+	}
+	return 0
+}

internal/caches/storage_file_test.go

@@ -62,7 +62,7 @@ func TestFileStorage_OpenWriter(t *testing.T) {
 
 	header := []byte("Header")
 	body := []byte("This is Body")
-	writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, false)
+	writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, -1, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -100,7 +100,7 @@ func TestFileStorage_OpenWriter_Partial(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, true)
+	writer, err := storage.OpenWriter("my-key", time.Now().Unix()+86400, 200, -1, -1, -1, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -139,7 +139,7 @@ func TestFileStorage_OpenWriter_HTTP(t *testing.T) {
 		t.Log(time.Since(now).Seconds()*1000, "ms")
 	}()
 
-	writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, -1, false)
+	writer, err := storage.OpenWriter("my-http-response", time.Now().Unix()+86400, 200, -1, -1, -1, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,7 +212,7 @@ func TestFileStorage_Concurrent_Open_DifferentFile(t *testing.T) {
 		go func(i int) {
 			defer wg.Done()
 
-			writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, false)
+			writer, err := storage.OpenWriter("abc"+strconv.Itoa(i), time.Now().Unix()+3600, 200, -1, -1, -1, false)
 			if err != nil {
 				if err != ErrFileIsWriting {
 					t.Error(err)
@@ -267,7 +267,7 @@ func TestFileStorage_Concurrent_Open_SameFile(t *testing.T) {
 		go func(i int) {
 			defer wg.Done()
 
-			writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, false)
+			writer, err := storage.OpenWriter("abc"+strconv.Itoa(0), time.Now().Unix()+3600, 200, -1, -1, -1, false)
 			if err != nil {
 				if err != ErrFileIsWriting {
 					t.Error(err)
@@ -522,7 +522,7 @@ func TestFileStorage_DecodeFile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, path := storage.keyPath("my-key")
+	_, path, _ := storage.keyPath("my-key")
 	t.Log(path)
 }
 
@@ -569,6 +569,6 @@ func BenchmarkFileStorage_KeyPath(b *testing.B) {
 	}
 
 	for i := 0; i < b.N; i++ {
-		_, _ = storage.keyPath(strconv.Itoa(i))
+		_, _, _ = storage.keyPath(strconv.Itoa(i))
 	}
 }

internal/caches/storage_interface.go

@@ -14,10 +14,10 @@ type StorageInterface interface {
 
 	// OpenWriter 打开缓存写入器等待写入
 	// size 和 maxSize 可能为-1
-	OpenWriter(key string, expiresAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error)
+	OpenWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error)
 
 	// OpenFlushWriter 打开从其他媒介直接刷入的写入器
-	OpenFlushWriter(key string, expiresAt int64, status int) (Writer, error)
+	OpenFlushWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64) (Writer, error)
 
 	// Delete 删除某个键值对应的缓存
 	Delete(key string) error

internal/caches/storage_memory.go

@@ -1,7 +1,6 @@
 package caches
 
 import (
-	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -17,6 +16,7 @@ import (
 	"math"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -149,7 +149,7 @@ func (this *MemoryStorage) OpenReader(key string, useStale bool, isPartial bool)
 }
 
 // OpenWriter 打开缓存写入器等待写入
-func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, size int64, maxSize int64, isPartial bool) (Writer, error) {
+func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, headerSize int, bodySize int64, maxSize int64, isPartial bool) (Writer, error) {
 	if this.ignoreKeys.Has(key) {
 		return nil, ErrEntityTooLarge
 	}
@@ -158,15 +158,15 @@ func (this *MemoryStorage) OpenWriter(key string, expiredAt int64, status int, s
 	if isPartial {
 		return nil, ErrFileIsWriting
 	}
-	return this.openWriter(key, expiredAt, status, size, maxSize, true)
+	return this.openWriter(key, expiredAt, status, headerSize, bodySize, maxSize, true)
 }
 
 // OpenFlushWriter 打开从其他媒介直接刷入的写入器
-func (this *MemoryStorage) OpenFlushWriter(key string, expiresAt int64, status int) (Writer, error) {
-	return this.openWriter(key, expiresAt, status, -1, -1, true)
+func (this *MemoryStorage) OpenFlushWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64) (Writer, error) {
+	return this.openWriter(key, expiresAt, status, headerSize, bodySize, -1, true)
 }
 
-func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, size int64, maxSize int64, isDirty bool) (Writer, error) {
+func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, headerSize int, bodySize int64, maxSize int64, isDirty bool) (Writer, error) {
 	// 待写入队列是否已满
 	if isDirty &&
 		this.parentStorage != nil &&
@@ -207,10 +207,10 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, s
 		return nil, NewCapacityError("write memory cache failed: too many keys in cache storage")
 	}
 	capacityBytes := this.memoryCapacityBytes()
-	if size < 0 {
-		size = 0
+	if bodySize < 0 {
+		bodySize = 0
 	}
-	if capacityBytes > 0 && capacityBytes <= this.totalSize+size {
+	if capacityBytes > 0 && capacityBytes <= this.totalSize+bodySize {
 		return nil, NewCapacityError("write memory cache failed: over memory size: " + strconv.FormatInt(capacityBytes, 10) + ", current size: " + strconv.FormatInt(this.totalSize, 10) + " bytes")
 	}
 
@@ -230,10 +230,10 @@ func (this *MemoryStorage) openWriter(key string, expiresAt int64, status int, s
 
 // Delete 删除某个键值对应的缓存
 func (this *MemoryStorage) Delete(key string) error {
-	hash := this.hash(key)
+	var hash = this.hash(key)
 	this.locker.Lock()
 	delete(this.valuesMap, hash)
-	_ = this.list.Remove(fmt.Sprintf("%d", hash))
+	_ = this.list.Remove(types.String(hash))
 	this.locker.Unlock()
 	return nil
 }
@@ -263,6 +263,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
 	// 目录
 	if urlType == "dir" {
 		for _, key := range keys {
+			// 检查是否有通配符 http(s)://*.example.com
+			var schemeIndex = strings.Index(key, "://")
+			if schemeIndex > 0 {
+				var keyRight = key[schemeIndex+3:]
+				if strings.HasPrefix(keyRight, "*.") {
+					err := this.list.CleanMatchPrefix(key)
+					if err != nil {
+						return err
+					}
+					continue
+				}
+			}
+
 			err := this.list.CleanPrefix(key)
 			if err != nil {
 				return err
@@ -273,6 +286,19 @@ func (this *MemoryStorage) Purge(keys []string, urlType string) error {
 
 	// URL
 	for _, key := range keys {
+		// 检查是否有通配符 http(s)://*.example.com
+		var schemeIndex = strings.Index(key, "://")
+		if schemeIndex > 0 {
+			var keyRight = key[schemeIndex+3:]
+			if strings.HasPrefix(keyRight, "*.") {
+				err := this.list.CleanMatchKey(key)
+				if err != nil {
+					return err
+				}
+				continue
+			}
+		}
+
 		err := this.Delete(key)
 		if err != nil {
 			return err
@@ -336,7 +362,12 @@ func (this *MemoryStorage) CanUpdatePolicy(newPolicy *serverconfigs.HTTPCachePol
 // AddToList 将缓存添加到列表
 func (this *MemoryStorage) AddToList(item *Item) {
 	item.MetaSize = int64(len(item.Key)) + 128 /** 128是我们评估的数据结构的长度 **/
-	hash := fmt.Sprintf("%d", this.hash(item.Key))
+	var hash = types.String(this.hash(item.Key))
+
+	if len(item.Host) == 0 {
+		item.Host = ParseHost(item.Key)
+	}
+
 	_ = this.list.Add(hash, item)
 }
 
@@ -433,7 +464,7 @@ func (this *MemoryStorage) startFlush() {
 	var statCount = 0
 	var writeDelayMS float64 = 0
 
-	for hash := range this.dirtyChan {
+	for key := range this.dirtyChan {
 		statCount++
 
 		if statCount == 100 {
@@ -455,7 +486,7 @@ func (this *MemoryStorage) startFlush() {
 			}
 		}
 
-		this.flushItem(hash)
+		this.flushItem(key)
 
 		if writeDelayMS > 0 {
 			time.Sleep(time.Duration(writeDelayMS) * time.Millisecond)
@@ -481,7 +512,7 @@ func (this *MemoryStorage) flushItem(key string) {
 		return
 	}
 
-	writer, err := this.parentStorage.OpenFlushWriter(key, item.ExpiresAt, item.Status)
+	writer, err := this.parentStorage.OpenFlushWriter(key, item.ExpiresAt, item.Status, len(item.HeaderValue), int64(len(item.BodyValue)))
 	if err != nil {
 		if !CanIgnoreErr(err) {
 			remotelogs.Error("CACHE", "flush items failed: open writer failed: "+err.Error())
@@ -513,6 +544,7 @@ func (this *MemoryStorage) flushItem(key string) {
 	this.parentStorage.AddToList(&Item{
 		Type:       writer.ItemType(),
 		Key:        key,
+		Host:       ParseHost(key),
 		ExpiredAt:  item.ExpiresAt,
 		HeaderSize: writer.HeaderSize(),
 		BodySize:   writer.BodySize(),
@@ -520,8 +552,6 @@ func (this *MemoryStorage) flushItem(key string) {
 
 	// 从内存中移除
 	_ = this.Delete(key)
-
-	return
 }
 
 func (this *MemoryStorage) memoryCapacityBytes() int64 {
@@ -544,7 +574,7 @@ func (this *MemoryStorage) memoryCapacityBytes() int64 {
 func (this *MemoryStorage) deleteWithoutLocker(key string) error {
 	hash := this.hash(key)
 	delete(this.valuesMap, hash)
-	_ = this.list.Remove(fmt.Sprintf("%d", hash))
+	_ = this.list.Remove(types.String(hash))
 	return nil
 }
 

internal/caches/storage_memory_test.go

@@ -14,15 +14,22 @@ import (
 )
 
 func TestMemoryStorage_OpenWriter(t *testing.T) {
-	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
+	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
 
-	writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
+	writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, -1, false)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if err != nil {
 		t.Fatal(err)
 	}
 	_, _ = writer.WriteHeader([]byte("Header"))
 	_, _ = writer.Write([]byte("Hello"))
 	_, _ = writer.Write([]byte(", World"))
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
 	t.Log(storage.valuesMap)
 
 	{
@@ -30,6 +37,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
 		if err != nil {
 			if err == ErrNotFound {
 				t.Log("not found: abc")
+				return
 			} else {
 				t.Fatal(err)
 			}
@@ -63,7 +71,7 @@ func TestMemoryStorage_OpenWriter(t *testing.T) {
 		}
 	}
 
-	writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
+	writer, err = storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, -1, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -102,21 +110,29 @@ func TestMemoryStorage_OpenReaderLock(t *testing.T) {
 }
 
 func TestMemoryStorage_Delete(t *testing.T) {
-	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
+	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
 	{
-		writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc", time.Now().Unix()+60, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		t.Log(len(storage.valuesMap))
 	}
 	{
-		writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc1", time.Now().Unix()+60, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		t.Log(len(storage.valuesMap))
 	}
 	_ = storage.Delete("abc1")
@@ -124,14 +140,18 @@ func TestMemoryStorage_Delete(t *testing.T) {
 }
 
 func TestMemoryStorage_Stat(t *testing.T) {
-	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
+	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
 	expiredAt := time.Now().Unix() + 60
 	{
-		writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		t.Log(len(storage.valuesMap))
 		storage.AddToList(&Item{
 			Key:       "abc",
@@ -140,11 +160,15 @@ func TestMemoryStorage_Stat(t *testing.T) {
 		})
 	}
 	{
-		writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		t.Log(len(storage.valuesMap))
 		storage.AddToList(&Item{
 			Key:       "abc1",
@@ -161,14 +185,18 @@ func TestMemoryStorage_Stat(t *testing.T) {
 }
 
 func TestMemoryStorage_CleanAll(t *testing.T) {
-	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
-	expiredAt := time.Now().Unix() + 60
+	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
+	var expiredAt = time.Now().Unix() + 60
 	{
-		writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		storage.AddToList(&Item{
 			Key:       "abc",
 			BodySize:  5,
@@ -176,11 +204,15 @@ func TestMemoryStorage_CleanAll(t *testing.T) {
 		})
 	}
 	{
-		writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		storage.AddToList(&Item{
 			Key:       "abc1",
 			BodySize:  5,
@@ -199,11 +231,15 @@ func TestMemoryStorage_Purge(t *testing.T) {
 	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
 	expiredAt := time.Now().Unix() + 60
 	{
-		writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc", expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		storage.AddToList(&Item{
 			Key:       "abc",
 			BodySize:  5,
@@ -211,11 +247,15 @@ func TestMemoryStorage_Purge(t *testing.T) {
 		})
 	}
 	{
-		writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter("abc1", expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		storage.AddToList(&Item{
 			Key:       "abc1",
 			BodySize:  5,
@@ -231,7 +271,7 @@ func TestMemoryStorage_Purge(t *testing.T) {
 }
 
 func TestMemoryStorage_Expire(t *testing.T) {
-	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
+	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{
 		MemoryAutoPurgeInterval: 5,
 	}, nil)
 	err := storage.Init()
@@ -242,11 +282,15 @@ func TestMemoryStorage_Expire(t *testing.T) {
 	for i := 0; i < 1000; i++ {
 		expiredAt := time.Now().Unix() + int64(rands.Int(0, 60))
 		key := "abc" + strconv.Itoa(i)
-		writer, err := storage.OpenWriter(key, expiredAt, 200, -1, -1, false)
+		writer, err := storage.OpenWriter(key, expiredAt, 200, -1, -1, -1, false)
 		if err != nil {
 			t.Fatal(err)
 		}
 		_, _ = writer.Write([]byte("Hello"))
+		err = writer.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
 		storage.AddToList(&Item{
 			Key:       key,
 			BodySize:  5,
@@ -257,7 +301,7 @@ func TestMemoryStorage_Expire(t *testing.T) {
 }
 
 func TestMemoryStorage_Locker(t *testing.T) {
-	storage := NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
+	var storage = NewMemoryStorage(&serverconfigs.HTTPCachePolicy{}, nil)
 	err := storage.Init()
 	if err != nil {
 		t.Fatal(err)

internal/caches/utils.go

@@ -0,0 +1,30 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"net"
+	"strings"
+)
+
+func ParseHost(key string) string {
+	var schemeIndex = strings.Index(key, "://")
+	if schemeIndex <= 0 {
+		return ""
+	}
+
+	var firstSlashIndex = strings.Index(key[schemeIndex+3:], "/")
+	if firstSlashIndex <= 0 {
+		return ""
+	}
+
+	var host = key[schemeIndex+3 : schemeIndex+3+firstSlashIndex]
+
+	hostPart, _, err := net.SplitHostPort(host)
+	if err == nil && len(hostPart) > 0 {
+		host = configutils.QuoteIP(hostPart)
+	}
+
+	return host
+}

internal/caches/utils_test.go

@@ -0,0 +1,51 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package caches_test
+
+import (
+	"fmt"
+	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/cespare/xxhash"
+	"github.com/iwind/TeaGo/types"
+	"strconv"
+	"testing"
+)
+
+func TestParseHost(t *testing.T) {
+	for _, u := range []string{
+		"https://goedge.cn/hello/world",
+		"https://goedge.cn:8080/hello/world",
+		"https://goedge.cn/hello/world?v=1&t=123",
+		"https://[::1]:1234/hello/world?v=1&t=123",
+		"https://[::1]/hello/world?v=1&t=123",
+		"https://127.0.0.1/hello/world?v=1&t=123",
+		"https:/hello/world?v=1&t=123",
+		"123456",
+	} {
+		t.Log(u, "=>", caches.ParseHost(u))
+	}
+}
+
+func TestUintString(t *testing.T) {
+	t.Log(strconv.FormatUint(xxhash.Sum64String("https://goedge.cn/"), 10))
+	t.Log(strconv.FormatUint(123456789, 10))
+	t.Log(fmt.Sprintf("%d", 1234567890123))
+}
+
+func BenchmarkUint_String(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = strconv.FormatUint(1234567890123, 10)
+	}
+}
+
+func BenchmarkUint_String2(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = types.String(1234567890123)
+	}
+}
+
+func BenchmarkUint_String3(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_ = fmt.Sprintf("%d", 1234567890123)
+	}
+}

internal/caches/writer_file.go

@@ -11,25 +11,32 @@ import (
 )
 
 type FileWriter struct {
-	storage    StorageInterface
-	rawWriter  *os.File
-	key        string
-	headerSize int64
-	bodySize   int64
-	expiredAt  int64
-	maxSize    int64
-	endFunc    func()
-	once       sync.Once
+	storage   StorageInterface
+	rawWriter *os.File
+	key       string
+
+	metaHeaderSize int
+	headerSize     int64
+
+	metaBodySize int64 // 写入前的内容长度
+	bodySize     int64
+
+	expiredAt int64
+	maxSize   int64
+	endFunc   func()
+	once      sync.Once
 }
 
-func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, expiredAt int64, maxSize int64, endFunc func()) *FileWriter {
+func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, maxSize int64, endFunc func()) *FileWriter {
 	return &FileWriter{
-		storage:   storage,
-		key:       key,
-		rawWriter: rawWriter,
-		expiredAt: expiredAt,
-		maxSize:   maxSize,
-		endFunc:   endFunc,
+		storage:        storage,
+		key:            key,
+		rawWriter:      rawWriter,
+		expiredAt:      expiredAt,
+		maxSize:        maxSize,
+		endFunc:        endFunc,
+		metaHeaderSize: metaHeaderSize,
+		metaBodySize:   metaBodySize,
 	}
 }
 
@@ -45,7 +52,10 @@ func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
 
 // WriteHeaderLength 写入Header长度数据
 func (this *FileWriter) WriteHeaderLength(headerLength int) error {
-	bytes4 := make([]byte, 4)
+	if this.metaHeaderSize > 0 && this.metaHeaderSize == headerLength {
+		return nil
+	}
+	var bytes4 = make([]byte, 4)
 	binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
 	_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength, io.SeekStart)
 	if err != nil {
@@ -88,7 +98,10 @@ func (this *FileWriter) WriteAt(offset int64, data []byte) error {
 
 // WriteBodyLength 写入Body长度数据
 func (this *FileWriter) WriteBodyLength(bodyLength int64) error {
-	bytes8 := make([]byte, 8)
+	if this.metaBodySize >= 0 && bodyLength == this.metaBodySize {
+		return nil
+	}
+	var bytes8 = make([]byte, 8)
 	binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
 	_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength, io.SeekStart)
 	if err != nil {
@@ -109,7 +122,7 @@ func (this *FileWriter) Close() error {
 		this.endFunc()
 	})
 
-	path := this.rawWriter.Name()
+	var path = this.rawWriter.Name()
 
 	err := this.WriteHeaderLength(types.Int(this.headerSize))
 	if err != nil {

internal/caches/writer_partial_file.go

@@ -11,13 +11,18 @@ import (
 )
 
 type PartialFileWriter struct {
-	rawWriter  *os.File
-	key        string
-	headerSize int64
-	bodySize   int64
-	expiredAt  int64
-	endFunc    func()
-	once       sync.Once
+	rawWriter *os.File
+	key       string
+
+	metaHeaderSize int
+	headerSize     int64
+
+	metaBodySize int64
+	bodySize     int64
+
+	expiredAt int64
+	endFunc   func()
+	once      sync.Once
 
 	isNew      bool
 	isPartial  bool
@@ -27,17 +32,19 @@ type PartialFileWriter struct {
 	rangePath string
 }
 
-func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
+func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
 	return &PartialFileWriter{
-		key:        key,
-		rawWriter:  rawWriter,
-		expiredAt:  expiredAt,
-		endFunc:    endFunc,
-		isNew:      isNew,
-		isPartial:  isPartial,
-		bodyOffset: bodyOffset,
-		ranges:     ranges,
-		rangePath:  partialRangesFilePath(rawWriter.Name()),
+		key:            key,
+		rawWriter:      rawWriter,
+		expiredAt:      expiredAt,
+		endFunc:        endFunc,
+		isNew:          isNew,
+		isPartial:      isPartial,
+		bodyOffset:     bodyOffset,
+		ranges:         ranges,
+		rangePath:      partialRangesFilePath(rawWriter.Name()),
+		metaHeaderSize: metaHeaderSize,
+		metaBodySize:   metaBodySize,
 	}
 }
 
@@ -71,7 +78,11 @@ func (this *PartialFileWriter) AppendHeader(data []byte) error {
 
 // WriteHeaderLength 写入Header长度数据
 func (this *PartialFileWriter) WriteHeaderLength(headerLength int) error {
-	bytes4 := make([]byte, 4)
+	if this.metaHeaderSize > 0 && this.metaHeaderSize == headerLength {
+		return nil
+	}
+
+	var bytes4 = make([]byte, 4)
 	binary.BigEndian.PutUint32(bytes4, uint32(headerLength))
 	_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength, io.SeekStart)
 	if err != nil {
@@ -110,8 +121,13 @@ func (this *PartialFileWriter) WriteAt(offset int64, data []byte) error {
 	}
 
 	if this.bodyOffset == 0 {
-		this.bodyOffset = SizeMeta + int64(len(this.key)) + this.headerSize
+		var keyLength = 0
+		if this.ranges.Version == 0 { // 以往的版本包含有Key
+			keyLength = len(this.key)
+		}
+		this.bodyOffset = SizeMeta + int64(keyLength) + this.headerSize
 	}
+
 	_, err := this.rawWriter.WriteAt(data, this.bodyOffset+offset)
 	if err != nil {
 		return err
@@ -129,7 +145,10 @@ func (this *PartialFileWriter) SetBodyLength(bodyLength int64) {
 
 // WriteBodyLength 写入Body长度数据
 func (this *PartialFileWriter) WriteBodyLength(bodyLength int64) error {
-	bytes8 := make([]byte, 8)
+	if this.metaBodySize > 0 && this.metaBodySize == bodyLength {
+		return nil
+	}
+	var bytes8 = make([]byte, 8)
 	binary.BigEndian.PutUint64(bytes8, uint64(bodyLength))
 	_, err := this.rawWriter.Seek(SizeExpiresAt+SizeStatus+SizeURLLength+SizeHeaderLength, io.SeekStart)
 	if err != nil {
@@ -150,8 +169,11 @@ func (this *PartialFileWriter) Close() error {
 		this.endFunc()
 	})
 
+	this.ranges.BodySize = this.bodySize
 	err := this.ranges.WriteToFile(this.rangePath)
 	if err != nil {
+		_ = this.rawWriter.Close()
+		this.remove()
 		return err
 	}
 

internal/caches/writer_partial_file_test.go

@@ -26,8 +26,8 @@ func TestPartialFileWriter_Write(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	var ranges = caches.NewPartialRanges()
-	var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, true, true, 0, ranges, func() {
+	var ranges = caches.NewPartialRanges(0)
+	var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, -1, -1, true, true, 0, ranges, func() {
 		t.Log("end")
 	})
 	_, err = writer.WriteHeader([]byte("header"))

internal/configs/api_config.go

@@ -9,11 +9,15 @@ import (
 // APIConfig 节点API配置
 type APIConfig struct {
 	RPC struct {
-		Endpoints     []string `yaml:"endpoints"`
-		DisableUpdate bool     `yaml:"disableUpdate"`
-	} `yaml:"rpc"`
-	NodeId string `yaml:"nodeId"`
-	Secret string `yaml:"secret"`
+		Endpoints     []string `yaml:"endpoints" json:"endpoints"`
+		DisableUpdate bool     `yaml:"disableUpdate" json:"disableUpdate"`
+	} `yaml:"rpc" json:"rpc"`
+	NodeId string `yaml:"nodeId" json:"nodeId"`
+	Secret string `yaml:"secret" json:"secret"`
+}
+
+func NewAPIConfig() *APIConfig {
+	return &APIConfig{}
 }
 
 func LoadAPIConfig() (*APIConfig, error) {

internal/configs/cluster_config.go

@@ -3,9 +3,9 @@ package configs
 // ClusterConfig 集群配置
 type ClusterConfig struct {
 	RPC struct {
-		Endpoints     []string `yaml:"endpoints"`
-		DisableUpdate bool     `yaml:"disableUpdate"`
-	} `yaml:"rpc"`
-	ClusterId string `yaml:"clusterId"`
-	Secret    string `yaml:"secret"`
+		Endpoints     []string `yaml:"endpoints" json:"endpoints"`
+		DisableUpdate bool     `yaml:"disableUpdate" json:"disableUpdate"`
+	} `yaml:"rpc" json:"rpc"`
+	ClusterId string `yaml:"clusterId" json:"clusterId"`
+	Secret    string `yaml:"secret" json:"secret"`
 }

internal/configs/locker.go

@@ -1,5 +0,0 @@
-package configs
-
-import "sync"
-
-var sharedLocker = &sync.RWMutex{}

internal/conns/linger.go

@@ -0,0 +1,7 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package conns
+
+type LingerConn interface {
+	SetLinger(sec int) error
+}

internal/conns/map.go

@@ -0,0 +1,123 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package conns
+
+import (
+	"net"
+	"sync"
+)
+
+var SharedMap = NewMap()
+
+type Map struct {
+	m map[string]map[int]net.Conn // ip => { port => Conn  }
+
+	locker sync.RWMutex
+}
+
+func NewMap() *Map {
+	return &Map{
+		m: map[string]map[int]net.Conn{},
+	}
+}
+
+func (this *Map) Add(conn net.Conn) {
+	if conn == nil {
+		return
+	}
+	tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
+	if !ok {
+		return
+	}
+
+	var ip = tcpAddr.IP.String()
+	var port = tcpAddr.Port
+
+	this.locker.Lock()
+	defer this.locker.Unlock()
+	connMap, ok := this.m[ip]
+	if !ok {
+		this.m[ip] = map[int]net.Conn{port: conn}
+	} else {
+		connMap[port] = conn
+	}
+}
+
+func (this *Map) Remove(conn net.Conn) {
+	if conn == nil {
+		return
+	}
+	tcpAddr, ok := conn.RemoteAddr().(*net.TCPAddr)
+	if !ok {
+		return
+	}
+
+	var ip = tcpAddr.IP.String()
+	var port = tcpAddr.Port
+
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	connMap, ok := this.m[ip]
+	if !ok {
+		return
+	}
+	delete(connMap, port)
+
+	if len(connMap) == 0 {
+		delete(this.m, ip)
+	}
+}
+
+func (this *Map) CountIPConns(ip string) int {
+	this.locker.RLock()
+	var l = len(this.m[ip])
+	this.locker.RUnlock()
+	return l
+}
+
+func (this *Map) CloseIPConns(ip string) {
+	var conns = []net.Conn{}
+
+	this.locker.RLock()
+	connMap, ok := this.m[ip]
+
+	// 复制，防止在Close时产生并发冲突
+	if ok {
+		for _, conn := range connMap {
+			conns = append(conns, conn)
+		}
+	}
+
+	// 需要在Close之前结束，防止死循环
+	this.locker.RUnlock()
+
+	if ok {
+		for _, conn := range conns {
+			// 设置Linger
+			lingerConn, isLingerConn := conn.(LingerConn)
+			if isLingerConn {
+				_ = lingerConn.SetLinger(0)
+			}
+
+			// 关闭
+			_ = conn.Close()
+		}
+
+		// 这里不需要从 m 中删除，因为关闭时会自然触发回调
+	}
+}
+
+func (this *Map) AllConns() []net.Conn {
+	this.locker.RLock()
+	defer this.locker.RUnlock()
+
+	var result = []net.Conn{}
+	for _, m := range this.m {
+		for _, connInfo := range m {
+			result = append(result, connInfo)
+		}
+	}
+
+	return result
+}

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "0.5.2"
+	Version = "0.6.1"
 
 	ProductName = "Edge Node"
 	ProcessName = "edge-node"

internal/firewalls/ddos_protection.go

@@ -1,6 +1,5 @@
 // Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 //go:build linux
-// +build linux
 
 package firewalls
 
@@ -14,6 +13,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/lists"
 	"github.com/iwind/TeaGo/types"
@@ -21,6 +21,7 @@ import (
 	"net"
 	"os/exec"
 	"strings"
+	"time"
 )
 
 var SharedDDoSProtectionManager = NewDDoSProtectionManager()
@@ -53,19 +54,13 @@ func init() {
 
 // DDoSProtectionManager DDoS防护
 type DDoSProtectionManager struct {
-	nftPath string
-
 	lastAllowIPList []string
 	lastConfig      []byte
 }
 
 // NewDDoSProtectionManager 获取新对象
 func NewDDoSProtectionManager() *DDoSProtectionManager {
-	nftPath, _ := exec.LookPath("nft")
-
-	return &DDoSProtectionManager{
-		nftPath: nftPath,
-	}
+	return &DDoSProtectionManager{}
 }
 
 // Apply 应用配置
@@ -75,7 +70,7 @@ func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) e
 	nodeConfig, _ := nodeconfigs.SharedNodeConfig()
 	if nodeConfig != nil {
 		var allowIPList = nodeConfig.AllowedIPs
-		if !utils.ContainsSameStrings(allowIPList, this.lastAllowIPList) {
+		if !utils.EqualStrings(allowIPList, this.lastAllowIPList) {
 			allowIPListChanged = true
 			this.lastAllowIPList = allowIPList
 		}
@@ -91,11 +86,14 @@ func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) e
 	}
 	remotelogs.Println("FIREWALL", "change DDoS protection config")
 
-	if len(this.nftPath) == 0 {
+	if len(this.nftExe()) == 0 {
 		return errors.New("can not find nft command")
 	}
 
 	if nftablesInstance == nil {
+		if config == nil || !config.IsOn() {
+			return nil
+		}
 		return errors.New("nftables instance should not be nil")
 	}
 
@@ -154,6 +152,11 @@ func (this *DDoSProtectionManager) Apply(config *ddosconfigs.ProtectionConfig) e
 
 // 添加TCP规则
 func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig) error {
+	var nftExe = this.nftExe()
+	if len(nftExe) == 0 {
+		return nil
+	}
+
 	// 检查nft版本不能小于0.9
 	if len(nftablesInstance.version) > 0 && stringutil.VersionCompare("0.9", nftablesInstance.version) > 0 {
 		return nil
@@ -195,14 +198,31 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
 			}
 		}
 
-		// new connections rate
-		var newConnectionsRate = tcpConfig.NewConnectionsRate
-		if newConnectionsRate <= 0 {
-			newConnectionsRate = nodeconfigs.DefaultTCPNewConnectionsRate
-			if newConnectionsRate <= 0 {
-				newConnectionsRate = 100000
+		// new connections rate (minutely)
+		var newConnectionsMinutelyRate = tcpConfig.NewConnectionsMinutelyRate
+		if newConnectionsMinutelyRate <= 0 {
+			newConnectionsMinutelyRate = nodeconfigs.DefaultTCPNewConnectionsMinutelyRate
+			if newConnectionsMinutelyRate <= 0 {
+				newConnectionsMinutelyRate = 100000
 			}
 		}
+		var newConnectionsMinutelyRateBlockTimeout = tcpConfig.NewConnectionsMinutelyRateBlockTimeout
+		if newConnectionsMinutelyRateBlockTimeout < 0 {
+			newConnectionsMinutelyRateBlockTimeout = 0
+		}
+
+		// new connections rate (secondly)
+		var newConnectionsSecondlyRate = tcpConfig.NewConnectionsSecondlyRate
+		if newConnectionsSecondlyRate <= 0 {
+			newConnectionsSecondlyRate = nodeconfigs.DefaultTCPNewConnectionsSecondlyRate
+			if newConnectionsSecondlyRate <= 0 {
+				newConnectionsSecondlyRate = 10000
+			}
+		}
+		var newConnectionsSecondlyRateBlockTimeout = tcpConfig.NewConnectionsSecondlyRateBlockTimeout
+		if newConnectionsSecondlyRateBlockTimeout < 0 {
+			newConnectionsSecondlyRateBlockTimeout = 0
+		}
 
 		// 检查是否有变化
 		var hasChanges = false
@@ -215,7 +235,11 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
 				hasChanges = true
 				break
 			}
-			if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}) {
+			if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}) {
+				hasChanges = true
+				break
+			}
+			if !this.existsRule(oldRules, []string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}) {
 				hasChanges = true
 				break
 			}
@@ -242,33 +266,61 @@ func (this *DDoSProtectionManager) addTCPRules(tcpConfig *ddosconfigs.TCPConfig)
 		// 添加新规则
 		for _, port := range ports {
 			if maxConnections > 0 {
-				var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "count", "over", types.String(maxConnections), "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnections", types.String(maxConnections)}))
-				var stderr = &bytes.Buffer{}
-				cmd.Stderr = stderr
+				var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "count", "over", types.String(maxConnections), "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnections", types.String(maxConnections)}))
+				cmd.WithStderr()
 				err := cmd.Run()
 				if err != nil {
-					return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
+					return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
 				}
 			}
 
+			// TODO 让用户选择是drop还是reject
 			if maxConnectionsPerIP > 0 {
-				var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
-				var stderr = &bytes.Buffer{}
-				cmd.Stderr = stderr
+				var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "meter", "meter-"+protocol+"-"+types.String(port)+"-max-connections", "{ "+protocol+" saddr ct count over "+types.String(maxConnectionsPerIP)+" }", "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "maxConnectionsPerIP", types.String(maxConnectionsPerIP)}))
+				cmd.WithStderr()
 				err := cmd.Run()
 				if err != nil {
-					return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
+					return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
 				}
 			}
 
-			if newConnectionsRate > 0 {
-				// TODO 思考是否有惩罚机制
-				var cmd = exec.Command(this.nftPath, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsRate)+"/minute burst "+types.String(newConnectionsRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsRate)}))
-				var stderr = &bytes.Buffer{}
-				cmd.Stderr = stderr
-				err := cmd.Run()
-				if err != nil {
-					return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + stderr.String() + ")")
+			// 超过一定速率就drop或者加入黑名单（分钟）
+			// TODO 让用户选择是drop还是reject
+			if newConnectionsMinutelyRate > 0 {
+				if newConnectionsMinutelyRateBlockTimeout > 0 {
+					var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsMinutelyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", types.String(newConnectionsMinutelyRate), types.String(newConnectionsMinutelyRateBlockTimeout)}))
+					cmd.WithStderr()
+					err := cmd.Run()
+					if err != nil {
+						return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
+					}
+				} else {
+					var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsMinutelyRate)+"/minute burst "+types.String(newConnectionsMinutelyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsRate", "0"}))
+					cmd.WithStderr()
+					err := cmd.Run()
+					if err != nil {
+						return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
+					}
+				}
+			}
+
+			// 超过一定速率就drop或者加入黑名单（秒）
+			// TODO 让用户选择是drop还是reject
+			if newConnectionsSecondlyRate > 0 {
+				if newConnectionsSecondlyRateBlockTimeout > 0 {
+					var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }", "add", "@deny_set", "{"+protocol+" saddr timeout "+types.String(newConnectionsSecondlyRateBlockTimeout)+"s}", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", types.String(newConnectionsSecondlyRate), types.String(newConnectionsSecondlyRateBlockTimeout)}))
+					cmd.WithStderr()
+					err := cmd.Run()
+					if err != nil {
+						return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
+					}
+				} else {
+					var cmd = executils.NewTimeoutCmd(10*time.Second, nftExe, "add", "rule", protocol, filter.Name, nftablesChainName, "tcp", "dport", types.String(port), "ct", "state", "new", "meter", "meter-"+protocol+"-"+types.String(port)+"-new-connections-secondly-rate", "{ "+protocol+" saddr limit rate over "+types.String(newConnectionsSecondlyRate)+"/second burst "+types.String(newConnectionsSecondlyRate+3)+" packets }" /**"add", "@deny_set", "{"+protocol+" saddr}",**/, "counter", "drop", "comment", this.encodeUserData([]string{"tcp", types.String(port), "newConnectionsSecondlyRate", "0"}))
+					cmd.WithStderr()
+					err := cmd.Run()
+					if err != nil {
+						return errors.New("add nftables rule '" + cmd.String() + "' failed: " + err.Error() + " (" + cmd.Stderr() + ")")
+					}
 				}
 			}
 		}
@@ -335,14 +387,14 @@ func (this *DDoSProtectionManager) decodeUserData(data []byte) []string {
 func (this *DDoSProtectionManager) removeOldTCPRules(chain *nftables.Chain, rules []*nftables.Rule) error {
 	for _, rule := range rules {
 		var pieces = this.decodeUserData(rule.UserData())
-		if len(pieces) != 4 {
+		if len(pieces) < 4 {
 			continue
 		}
 		if pieces[0] != "tcp" {
 			continue
 		}
 		switch pieces[2] {
-		case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate":
+		case "maxConnections", "maxConnectionsPerIP", "newConnectionsRate", "newConnectionsSecondlyRate":
 			err := chain.DeleteRule(rule)
 			if err != nil {
 				return err
@@ -500,3 +552,8 @@ func (this *DDoSProtectionManager) updateAllowIPList(allIPList []string) error {
 
 	return nil
 }
+
+func (this *DDoSProtectionManager) nftExe() string {
+	path, _ := exec.LookPath("nft")
+	return path
+}

internal/firewalls/ddos_protection_others.go

@@ -11,7 +11,6 @@ import (
 var SharedDDoSProtectionManager = NewDDoSProtectionManager()
 
 type DDoSProtectionManager struct {
-	nftPath string
 }
 
 func NewDDoSProtectionManager() *DDoSProtectionManager {

internal/firewalls/firewall_base.go

@@ -0,0 +1,47 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package firewalls
+
+import (
+	"github.com/iwind/TeaGo/types"
+	"strings"
+	"sync"
+	"time"
+)
+
+type BaseFirewall struct {
+	locker        sync.Mutex
+	latestIPTimes []string // [ip@time, ....]
+}
+
+// 检查是否在最近添加过
+func (this *BaseFirewall) checkLatestIP(ip string) bool {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	var expiredIndex = -1
+	for index, ipTime := range this.latestIPTimes {
+		var pieces = strings.Split(ipTime, "@")
+		var oldIP = pieces[0]
+		var oldTimestamp = pieces[1]
+		if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
+			expiredIndex = index
+			continue
+		}
+		if oldIP == ip {
+			return true
+		}
+	}
+
+	if expiredIndex > -1 {
+		this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
+	}
+
+	this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
+	const maxLen = 128
+	if len(this.latestIPTimes) > maxLen {
+		this.latestIPTimes = this.latestIPTimes[1:]
+	}
+
+	return false
+}

internal/firewalls/firewall_firewalld.go

@@ -4,27 +4,37 @@ package firewalls
 
 import (
 	"errors"
+	"github.com/TeaOSLab/EdgeNode/internal/conns"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/iwind/TeaGo/types"
 	"os/exec"
 	"strings"
+	"time"
 )
 
+type firewalldCmd struct {
+	cmd    *executils.Cmd
+	denyIP string
+}
+
 type Firewalld struct {
+	BaseFirewall
+
 	isReady  bool
 	exe      string
-	cmdQueue chan *exec.Cmd
+	cmdQueue chan *firewalldCmd
 }
 
 func NewFirewalld() *Firewalld {
 	var firewalld = &Firewalld{
-		cmdQueue: make(chan *exec.Cmd, 4096),
+		cmdQueue: make(chan *firewalldCmd, 4096),
 	}
 
 	path, err := exec.LookPath("firewall-cmd")
 	if err == nil && len(path) > 0 {
-		var cmd = exec.Command(path, "--state")
+		var cmd = executils.NewTimeoutCmd(3*time.Second, path, "--state")
 		err := cmd.Run()
 		if err == nil {
 			firewalld.exe = path
@@ -41,13 +51,19 @@ func NewFirewalld() *Firewalld {
 
 func (this *Firewalld) init() {
 	goman.New(func() {
-		for cmd := range this.cmdQueue {
+		for c := range this.cmdQueue {
+			var cmd = c.cmd
 			err := cmd.Run()
 			if err != nil {
 				if strings.HasPrefix(err.Error(), "Warning:") {
 					continue
 				}
 				remotelogs.Warn("FIREWALL", "run command failed '"+cmd.String()+"': "+err.Error())
+			} else {
+				// 关闭连接
+				if len(c.denyIP) > 0 {
+					conns.SharedMap.CloseIPConns(c.denyIP)
+				}
 			}
 		}
 	})
@@ -71,8 +87,8 @@ func (this *Firewalld) AllowPort(port int, protocol string) error {
 	if !this.isReady {
 		return nil
 	}
-	var cmd = exec.Command(this.exe, "--add-port="+types.String(port)+"/"+protocol)
-	this.pushCmd(cmd)
+	var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--add-port="+types.String(port)+"/"+protocol)
+	this.pushCmd(cmd, "")
 	return nil
 }
 
@@ -81,13 +97,13 @@ func (this *Firewalld) AllowPortRangesPermanently(portRanges [][2]int, protocol
 		var port = this.PortRangeString(portRange, protocol)
 
 		{
-			var cmd = exec.Command(this.exe, "--add-port="+port, "--permanent")
-			this.pushCmd(cmd)
+			var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--add-port="+port, "--permanent")
+			this.pushCmd(cmd, "")
 		}
 
 		{
-			var cmd = exec.Command(this.exe, "--add-port="+port)
-			this.pushCmd(cmd)
+			var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--add-port="+port)
+			this.pushCmd(cmd, "")
 		}
 	}
 
@@ -98,8 +114,8 @@ func (this *Firewalld) RemovePort(port int, protocol string) error {
 	if !this.isReady {
 		return nil
 	}
-	var cmd = exec.Command(this.exe, "--remove-port="+types.String(port)+"/"+protocol)
-	this.pushCmd(cmd)
+	var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--remove-port="+types.String(port)+"/"+protocol)
+	this.pushCmd(cmd, "")
 	return nil
 }
 
@@ -107,13 +123,13 @@ func (this *Firewalld) RemovePortRangePermanently(portRange [2]int, protocol str
 	var port = this.PortRangeString(portRange, protocol)
 
 	{
-		var cmd = exec.Command(this.exe, "--remove-port="+port, "--permanent")
-		this.pushCmd(cmd)
+		var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--remove-port="+port, "--permanent")
+		this.pushCmd(cmd, "")
 	}
 
 	{
-		var cmd = exec.Command(this.exe, "--remove-port="+port)
-		this.pushCmd(cmd)
+		var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, "--remove-port="+port)
+		this.pushCmd(cmd, "")
 	}
 
 	return nil
@@ -131,6 +147,12 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
 	if !this.isReady {
 		return nil
 	}
+
+	// 避免短时间内重复添加
+	if this.checkLatestIP(ip) {
+		return nil
+	}
+
 	var family = "ipv4"
 	if strings.Contains(ip, ":") {
 		family = "ipv6"
@@ -139,8 +161,8 @@ func (this *Firewalld) RejectSourceIP(ip string, timeoutSeconds int) error {
 	if timeoutSeconds > 0 {
 		args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
 	}
-	var cmd = exec.Command(this.exe, args...)
-	this.pushCmd(cmd)
+	var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, args...)
+	this.pushCmd(cmd, ip)
 	return nil
 }
 
@@ -148,6 +170,12 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) e
 	if !this.isReady {
 		return nil
 	}
+
+	// 避免短时间内重复添加
+	if async && this.checkLatestIP(ip) {
+		return nil
+	}
+
 	var family = "ipv4"
 	if strings.Contains(ip, ":") {
 		family = "ipv6"
@@ -156,12 +184,15 @@ func (this *Firewalld) DropSourceIP(ip string, timeoutSeconds int, async bool) e
 	if timeoutSeconds > 0 {
 		args = append(args, "--timeout="+types.String(timeoutSeconds)+"s")
 	}
-	var cmd = exec.Command(this.exe, args...)
+	var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, args...)
 	if async {
-		this.pushCmd(cmd)
+		this.pushCmd(cmd, ip)
 		return nil
 	}
 
+	// 关闭连接
+	defer conns.SharedMap.CloseIPConns(ip)
+
 	err := cmd.Run()
 	if err != nil {
 		return errors.New("run command failed '" + cmd.String() + "': " + err.Error())
@@ -173,21 +204,22 @@ func (this *Firewalld) RemoveSourceIP(ip string) error {
 	if !this.isReady {
 		return nil
 	}
+
 	var family = "ipv4"
 	if strings.Contains(ip, ":") {
 		family = "ipv6"
 	}
 	for _, action := range []string{"reject", "drop"} {
 		var args = []string{"--remove-rich-rule=rule family='" + family + "' source address='" + ip + "' " + action}
-		var cmd = exec.Command(this.exe, args...)
-		this.pushCmd(cmd)
+		var cmd = executils.NewTimeoutCmd(10*time.Second, this.exe, args...)
+		this.pushCmd(cmd, "")
 	}
 	return nil
 }
 
-func (this *Firewalld) pushCmd(cmd *exec.Cmd) {
+func (this *Firewalld) pushCmd(cmd *executils.Cmd, denyIP string) {
 	select {
-	case this.cmdQueue <- cmd:
+	case this.cmdQueue <- &firewalldCmd{cmd: cmd, denyIP: denyIP}:
 	default:
 		// we discard the command
 	}

internal/firewalls/firewall_nftables.go

@@ -5,13 +5,14 @@
 package firewalls
 
 import (
-	"bytes"
 	"errors"
+	"github.com/TeaOSLab/EdgeNode/internal/conns"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/iwind/TeaGo/types"
 	"net"
 	"os/exec"
@@ -100,6 +101,8 @@ func NewNFTablesFirewall() (*NFTablesFirewall, error) {
 }
 
 type NFTablesFirewall struct {
+	BaseFirewall
+
 	conn    *nftables.Conn
 	isReady bool
 	version string
@@ -344,6 +347,14 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
 		return errors.New("invalid ip '" + ip + "'")
 	}
 
+	// 尝试关闭连接
+	conns.SharedMap.CloseIPConns(ip)
+
+	// 避免短时间内重复添加
+	if async && this.checkLatestIP(ip) {
+		return nil
+	}
+
 	if async {
 		select {
 		case this.dropIPQueue <- &blockIPItem{
@@ -357,6 +368,9 @@ func (this *NFTablesFirewall) DropSourceIP(ip string, timeoutSeconds int, async
 		return nil
 	}
 
+	// 再次尝试关闭连接
+	defer conns.SharedMap.CloseIPConns(ip)
+
 	if strings.Contains(ip, ":") { // ipv6
 		if this.denyIPv6Set == nil {
 			return errors.New("ipv6 ip set is nil")
@@ -418,18 +432,49 @@ func (this *NFTablesFirewall) RemoveSourceIP(ip string) error {
 
 // 读取版本号
 func (this *NFTablesFirewall) readVersion(nftPath string) string {
-	var cmd = exec.Command(nftPath, "--version")
-	var output = &bytes.Buffer{}
-	cmd.Stdout = output
+	var cmd = executils.NewTimeoutCmd(10*time.Second, nftPath, "--version")
+	cmd.WithStdout()
 	err := cmd.Run()
 	if err != nil {
 		return ""
 	}
 
-	var outputString = output.String()
+	var outputString = cmd.Stdout()
 	var versionMatches = regexp.MustCompile(`nftables v([\d.]+)`).FindStringSubmatch(outputString)
 	if len(versionMatches) <= 1 {
 		return ""
 	}
 	return versionMatches[1]
 }
+
+// 检查是否在最近添加过
+func (this *NFTablesFirewall) existLatestIP(ip string) bool {
+	this.locker.Lock()
+	defer this.locker.Unlock()
+
+	var expiredIndex = -1
+	for index, ipTime := range this.latestIPTimes {
+		var pieces = strings.Split(ipTime, "@")
+		var oldIP = pieces[0]
+		var oldTimestamp = pieces[1]
+		if types.Int64(oldTimestamp) < time.Now().Unix()-3 /** 3秒外表示过期 **/ {
+			expiredIndex = index
+			continue
+		}
+		if oldIP == ip {
+			return true
+		}
+	}
+
+	if expiredIndex > -1 {
+		this.latestIPTimes = this.latestIPTimes[expiredIndex+1:]
+	}
+
+	this.latestIPTimes = append(this.latestIPTimes, ip+"@"+types.String(time.Now().Unix()))
+	const maxLen = 128
+	if len(this.latestIPTimes) > maxLen {
+		this.latestIPTimes = this.latestIPTimes[1:]
+	}
+
+	return false
+}

internal/firewalls/nftables/conn.go

@@ -1,6 +1,5 @@
 // Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 //go:build linux
-// +build linux
 
 package nftables
 

internal/firewalls/nftables/installer.go

@@ -0,0 +1,108 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package nftables
+
+import (
+	"errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/events"
+	"github.com/TeaOSLab/EdgeNode/internal/goman"
+	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
+	"github.com/iwind/TeaGo/logs"
+	"os"
+	"os/exec"
+	"runtime"
+	"time"
+)
+
+func init() {
+	events.On(events.EventReload, func() {
+		// linux only
+		if runtime.GOOS != "linux" {
+			return
+		}
+
+		nodeConfig, err := nodeconfigs.SharedNodeConfig()
+		if err != nil {
+			return
+		}
+
+		if nodeConfig == nil || !nodeConfig.AutoInstallNftables {
+			return
+		}
+
+		if os.Getgid() == 0 { // root user only
+			_, err := exec.LookPath("nft")
+			if err == nil {
+				return
+			}
+			goman.New(func() {
+				err := NewInstaller().Install()
+				if err != nil {
+					// 不需要传到API节点
+					logs.Println("[NFTABLES]install nftables failed: " + err.Error())
+				}
+			})
+		}
+	})
+}
+
+type Installer struct {
+}
+
+func NewInstaller() *Installer {
+	return &Installer{}
+}
+
+func (this *Installer) Install() error {
+	// linux only
+	if runtime.GOOS != "linux" {
+		return nil
+	}
+
+	// 检查是否已经存在
+	_, err := exec.LookPath("nft")
+	if err == nil {
+		return nil
+	}
+
+	var cmd *executils.Cmd
+
+	// check dnf
+	dnfExe, err := exec.LookPath("dnf")
+	if err == nil {
+		cmd = executils.NewCmd(dnfExe, "-y", "install", "nftables")
+	}
+
+	// check apt
+	if cmd == nil {
+		aptExe, err := exec.LookPath("apt")
+		if err == nil {
+			cmd = executils.NewCmd(aptExe, "install", "nftables")
+		}
+	}
+
+	// check yum
+	if cmd == nil {
+		yumExe, err := exec.LookPath("yum")
+		if err == nil {
+			cmd = executils.NewCmd(yumExe, "-y", "install", "nftables")
+		}
+	}
+
+	if cmd == nil {
+		return nil
+	}
+
+	cmd.WithTimeout(10 * time.Minute)
+	cmd.WithStderr()
+	err = cmd.Run()
+	if err != nil {
+		return errors.New(err.Error() + ": " + cmd.Stderr())
+	}
+
+	remotelogs.Println("NFTABLES", "installed nftables with command '"+cmd.String()+"' successfully")
+
+	return nil
+}

internal/firewalls/nftables/set_batch.go

@@ -1,4 +1,5 @@
 // Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build linux
 
 package nftables
 

internal/firewalls/nftables/set_test.go

@@ -1,4 +1,5 @@
 // Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build linux
 
 package nftables_test
 

internal/iplibrary/action_firewalld.go

@@ -1,11 +1,11 @@
 package iplibrary
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"os/exec"
 	"runtime"
 	"time"
@@ -13,9 +13,9 @@ import (
 
 // FirewalldAction Firewalld动作管理
 // 常用命令：
-//  - 查询列表： firewall-cmd --list-all
-//  - 添加IP：firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
-//  - 删除IP：firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
+//   - 查询列表： firewall-cmd --list-all
+//   - 添加IP：firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
+//   - 删除IP：firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.168.2.32' reject" --timeout=30s
 type FirewalldAction struct {
 	BaseAction
 
@@ -144,12 +144,11 @@ func (this *FirewalldAction) runActionSingleIP(action string, listType IPListTyp
 		// MAC OS直接返回
 		return nil
 	}
-	cmd := exec.Command(path, args...)
-	stderr := bytes.NewBuffer([]byte{})
-	cmd.Stderr = stderr
+	cmd := executils.NewTimeoutCmd(30*time.Second, path, args...)
+	cmd.WithStderr()
 	err = cmd.Run()
 	if err != nil {
-		return errors.New(err.Error() + ", output: " + string(stderr.Bytes()))
+		return errors.New(err.Error() + ", output: " + cmd.Stderr())
 	}
 	return nil
 }

internal/iplibrary/action_ipset.go

@@ -1,10 +1,10 @@
 package iplibrary
 
 import (
-	"bytes"
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/iwind/TeaGo/types"
 	"os/exec"
 	"runtime"
@@ -16,12 +16,12 @@ import (
 // IPSetAction IPSet动作
 // 相关命令：
 //   - 利用Firewalld管理set：
-//       - 添加：firewall-cmd --permanent --new-ipset=edge_ip_list --type=hash:ip --option="timeout=0"
-//       - 删除：firewall-cmd --permanent --delete-ipset=edge_ip_list
-//       - 重载：firewall-cmd --reload
-//       - firewalld+ipset: firewall-cmd --permanent --add-rich-rule="rule source ipset='edge_ip_list' reject"
+//   - 添加：firewall-cmd --permanent --new-ipset=edge_ip_list --type=hash:ip --option="timeout=0"
+//   - 删除：firewall-cmd --permanent --delete-ipset=edge_ip_list
+//   - 重载：firewall-cmd --reload
+//   - firewalld+ipset: firewall-cmd --permanent --add-rich-rule="rule source ipset='edge_ip_list' reject"
 //   - 利用IPTables管理set：
-//       - 添加：iptables -A INPUT -m set --match-set edge_ip_list src -j REJECT
+//   - 添加：iptables -A INPUT -m set --match-set edge_ip_list src -j REJECT
 //   - 添加Item：ipset add edge_ip_list 192.168.2.32 timeout 30
 //   - 删除Item: ipset del edge_ip_list 192.168.2.32
 //   - 创建set：ipset create edge_ip_list hash:ip timeout 0
@@ -30,16 +30,13 @@ import (
 type IPSetAction struct {
 	BaseAction
 
-	config   *firewallconfigs.FirewallActionIPSetConfig
-	errorBuf *bytes.Buffer
+	config *firewallconfigs.FirewallActionIPSetConfig
 
 	ipsetNotfound bool
 }
 
 func NewIPSetAction() *IPSetAction {
-	return &IPSetAction{
-		errorBuf: &bytes.Buffer{},
-	}
+	return &IPSetAction{}
 }
 
 func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) error {
@@ -68,14 +65,13 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			if len(listName) == 0 {
 				continue
 			}
-			var cmd = exec.Command(path, "create", listName, "hash:ip", "timeout", "0", "maxelem", "1000000")
-			var stderr = bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "create", listName, "hash:ip", "timeout", "0", "maxelem", "1000000")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				var output = stderr.Bytes()
-				if !bytes.Contains(output, []byte("already exists")) {
-					return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
+				var output = cmd.Stderr()
+				if !strings.Contains(output, "already exists") {
+					return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + output)
 				} else {
 					err = nil
 				}
@@ -87,14 +83,13 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			if len(listName) == 0 {
 				continue
 			}
-			var cmd = exec.Command(path, "create", listName, "hash:ip", "family", "inet6", "timeout", "0", "maxelem", "1000000")
-			var stderr = bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "create", listName, "hash:ip", "family", "inet6", "timeout", "0", "maxelem", "1000000")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				var output = stderr.Bytes()
-				if !bytes.Contains(output, []byte("already exists")) {
-					return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
+				var output = cmd.Stderr()
+				if !strings.Contains(output, "already exists") {
+					return errors.New("create ipset '" + listName + "': " + err.Error() + ", output: " + output)
 				} else {
 					err = nil
 				}
@@ -114,16 +109,15 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			if len(listName) == 0 {
 				continue
 			}
-			cmd := exec.Command(path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
-			stderr := bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=timeout=0", "--option=maxelem=1000000")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				output := stderr.Bytes()
-				if bytes.Contains(output, []byte("NAME_CONFLICT")) {
+				var output = cmd.Stderr()
+				if strings.Contains(output, "NAME_CONFLICT") {
 					err = nil
 				} else {
-					return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
+					return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + output)
 				}
 			}
 		}
@@ -133,16 +127,15 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			if len(listName) == 0 {
 				continue
 			}
-			cmd := exec.Command(path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=family=inet6", "--option=timeout=0", "--option=maxelem=1000000")
-			stderr := bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--new-ipset="+listName, "--type=hash:ip", "--option=family=inet6", "--option=timeout=0", "--option=maxelem=1000000")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				var output = stderr.Bytes()
-				if bytes.Contains(output, []byte("NAME_CONFLICT")) {
+				var output = cmd.Stderr()
+				if strings.Contains(output, "NAME_CONFLICT") {
 					err = nil
 				} else {
-					return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + string(output))
+					return errors.New("firewall-cmd add ipset '" + listName + "': " + err.Error() + ", output: " + output)
 				}
 			}
 		}
@@ -152,13 +145,11 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			if len(listName) == 0 {
 				continue
 			}
-			var cmd = exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' accept")
-			var stderr = bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' accept")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				var output = stderr.Bytes()
-				return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + string(output))
+				return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + cmd.Stderr())
 			}
 		}
 
@@ -167,25 +158,21 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			if len(listName) == 0 {
 				continue
 			}
-			var cmd = exec.Command(path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' reject")
-			var stderr = bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--permanent", "--add-rich-rule=rule source ipset='"+listName+"' reject")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				var output = stderr.Bytes()
-				return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + string(output))
+				return errors.New("firewall-cmd add rich rule '" + listName + "': " + err.Error() + ", output: " + cmd.Stderr())
 			}
 		}
 
 		// reload
 		{
-			cmd := exec.Command(path, "--reload")
-			stderr := bytes.NewBuffer([]byte{})
-			cmd.Stderr = stderr
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "--reload")
+			cmd.WithStderr()
 			err := cmd.Run()
 			if err != nil {
-				var output = stderr.Bytes()
-				return errors.New("firewall-cmd reload: " + err.Error() + ", output: " + string(output))
+				return errors.New("firewall-cmd reload: " + err.Error() + ", output: " + cmd.Stderr())
 			}
 		}
 	}
@@ -204,19 +191,17 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			}
 
 			// 检查规则是否存在
-			var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
 			err := cmd.Run()
 			var exists = err == nil
 
 			// 添加规则
 			if !exists {
-				var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
-				var stderr = bytes.NewBuffer([]byte{})
-				cmd.Stderr = stderr
+				var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "ACCEPT")
+				cmd.WithStderr()
 				err := cmd.Run()
 				if err != nil {
-					var output = stderr.Bytes()
-					return errors.New("iptables add rule: " + err.Error() + ", output: " + string(output))
+					return errors.New("iptables add rule: " + err.Error() + ", output: " + cmd.Stderr())
 				}
 			}
 		}
@@ -228,18 +213,16 @@ func (this *IPSetAction) Init(config *firewallconfigs.FirewallActionConfig) erro
 			}
 
 			// 检查规则是否存在
-			var cmd = exec.Command(path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
+			var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-C", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
 			err := cmd.Run()
 			var exists = err == nil
 
 			if !exists {
-				var cmd = exec.Command(path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
-				var stderr = bytes.NewBuffer([]byte{})
-				cmd.Stderr = stderr
+				var cmd = executils.NewTimeoutCmd(30*time.Second, path, "-A", "INPUT", "-m", "set", "--match-set", listName, "src", "-j", "REJECT")
+				cmd.WithStderr()
 				err := cmd.Run()
 				if err != nil {
-					var output = stderr.Bytes()
-					return errors.New("iptables add rule: " + err.Error() + ", output: " + string(output))
+					return errors.New("iptables add rule: " + err.Error() + ", output: " + cmd.Stderr())
 				}
 			}
 		}
@@ -361,12 +344,11 @@ func (this *IPSetAction) runActionSingleIP(action string, listType IPListType, i
 		return nil
 	}
 
-	this.errorBuf.Reset()
-	var cmd = exec.Command(path, args...)
-	cmd.Stderr = this.errorBuf
+	var cmd = executils.NewTimeoutCmd(30*time.Second, path, args...)
+	cmd.WithStderr()
 	err = cmd.Run()
 	if err != nil {
-		var errString = this.errorBuf.String()
+		var errString = cmd.Stderr()
 		if action == "deleteItem" && strings.Contains(errString, "not added") {
 			return nil
 		}

internal/iplibrary/action_iptables.go

@@ -1,20 +1,24 @@
 package iplibrary
 
 import (
-	"bytes"
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"os/exec"
 	"runtime"
+	"strings"
+	"time"
 )
 
 // IPTablesAction IPTables动作
 // 相关命令：
-//   iptables -A INPUT -s "192.168.2.32" -j ACCEPT
-//   iptables -A INPUT -s "192.168.2.32" -j REJECT
-//   iptables -D INPUT ...
-//   iptables -F INPUT
+//
+//	iptables -A INPUT -s "192.168.2.32" -j ACCEPT
+//	iptables -A INPUT -s "192.168.2.32" -j REJECT
+//	iptables -D INPUT ...
+//	iptables -F INPUT
 type IPTablesAction struct {
 	BaseAction
 
@@ -71,10 +75,16 @@ func (this *IPTablesAction) runAction(action string, listType IPListType, item *
 }
 
 func (this *IPTablesAction) runActionSingleIP(action string, listType IPListType, item *pb.IPItem) error {
+	// 暂时不支持ipv6
+	// TODO 将来支持ipv6
+	if utils.IsIPv6(item.IpFrom) {
+		return nil
+	}
+
 	if item.Type == "all" {
 		return nil
 	}
-	path := this.config.Path
+	var path = this.config.Path
 	var err error
 	if len(path) == 0 {
 		path, err = exec.LookPath("iptables")
@@ -85,6 +95,7 @@ func (this *IPTablesAction) runActionSingleIP(action string, listType IPListType
 			this.iptablesNotFound = true
 			return err
 		}
+		this.config.Path = path
 	}
 	iptablesAction := ""
 	switch action {
@@ -110,16 +121,15 @@ func (this *IPTablesAction) runActionSingleIP(action string, listType IPListType
 		return nil
 	}
 
-	cmd := exec.Command(path, args...)
-	stderr := bytes.NewBuffer([]byte{})
-	cmd.Stderr = stderr
+	var cmd = executils.NewTimeoutCmd(30*time.Second, path, args...)
+	cmd.WithStderr()
 	err = cmd.Run()
 	if err != nil {
-		output := stderr.Bytes()
-		if bytes.Contains(output, []byte("No chain/target/match")) {
+		var output = cmd.Stderr()
+		if strings.Contains(output, "No chain/target/match") {
 			err = nil
 		} else {
-			return errors.New(err.Error() + ", output: " + string(output))
+			return errors.New(err.Error() + ", output: " + output)
 		}
 	}
 	return nil

internal/iplibrary/action_manager.go

@@ -68,7 +68,7 @@ func (this *ActionManager) UpdateActions(actions []*firewallconfigs.FirewallActi
 				remotelogs.Error("IPLIBRARY/ACTION_MANAGER", "action "+strconv.FormatInt(newAction.Id, 10)+", type:"+newAction.Type+": "+err.Error())
 				continue
 			}
-			if bytes.Compare(newConfigJSON, oldConfigJSON) != 0 {
+			if !bytes.Equal(newConfigJSON, oldConfigJSON) {
 				_ = oldInstance.Close()
 
 				// 重新创建

internal/iplibrary/action_script.go

@@ -1,13 +1,13 @@
 package iplibrary
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
-	"os/exec"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"path/filepath"
+	"time"
 )
 
 // ScriptAction 脚本命令动作
@@ -45,25 +45,24 @@ func (this *ScriptAction) DeleteItem(listType IPListType, item *pb.IPItem) error
 
 func (this *ScriptAction) runAction(action string, listType IPListType, item *pb.IPItem) error {
 	// TODO 智能支持 .sh 脚本文件
-	cmd := exec.Command(this.config.Path)
-	cmd.Env = []string{
+	var cmd = executils.NewTimeoutCmd(30*time.Second, this.config.Path)
+	cmd.WithEnv([]string{
 		"ACTION=" + action,
 		"TYPE=" + item.Type,
 		"IP_FROM=" + item.IpFrom,
 		"IP_TO=" + item.IpTo,
 		"EXPIRED_AT=" + fmt.Sprintf("%d", item.ExpiredAt),
 		"LIST_TYPE=" + listType,
-	}
+	})
 	if len(this.config.Cwd) > 0 {
-		cmd.Dir = this.config.Cwd
+		cmd.WithDir(this.config.Cwd)
 	} else {
-		cmd.Dir = filepath.Dir(this.config.Path)
+		cmd.WithDir(filepath.Dir(this.config.Path))
 	}
-	stderr := bytes.NewBuffer([]byte{})
-	cmd.Stderr = stderr
+	cmd.WithStderr()
 	err := cmd.Run()
 	if err != nil {
-		return errors.New(err.Error() + ", output: " + string(stderr.Bytes()))
+		return errors.New(err.Error() + ", output: " + cmd.Stderr())
 	}
 	return nil
 }

internal/iplibrary/ip2Region.go

@@ -1,130 +0,0 @@
-// 源码改自：https://github.com/lionsoul2014/ip2region/blob/master/binding/golang/ip2region/ip2Region.go
-
-package iplibrary
-
-import (
-	"errors"
-	"os"
-	"strconv"
-	"strings"
-)
-
-const (
-	IndexBlockLength = 12
-)
-
-var err error
-
-type IP2Region struct {
-	headerSip []int64
-	headerPtr []int64
-	headerLen int64
-
-	// super block index info
-	firstIndexPtr int64
-	lastIndexPtr  int64
-	totalBlocks   int64
-
-	dbData []byte
-}
-
-type IpInfo struct {
-	CityId   int64
-	Country  string
-	Region   string
-	Province string
-	City     string
-	ISP      string
-}
-
-func (ip IpInfo) String() string {
-	return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
-}
-
-func getIpInfo(cityId int64, line []byte) *IpInfo {
-	lineSlice := strings.Split(string(line), "|")
-	ipInfo := &IpInfo{}
-	length := len(lineSlice)
-	ipInfo.CityId = cityId
-	if length < 5 {
-		for i := 0; i <= 5-length; i++ {
-			lineSlice = append(lineSlice, "")
-		}
-	}
-
-	ipInfo.Country = lineSlice[0]
-	ipInfo.Region = lineSlice[1]
-	ipInfo.Province = lineSlice[2]
-	ipInfo.City = lineSlice[3]
-	ipInfo.ISP = lineSlice[4]
-	return ipInfo
-}
-
-func NewIP2Region(path string) (*IP2Region, error) {
-	var region = &IP2Region{}
-	region.dbData, err = os.ReadFile(path)
-
-	if err != nil {
-		return nil, err
-	}
-
-	region.firstIndexPtr = region.ipLongAtOffset(0)
-	region.lastIndexPtr = region.ipLongAtOffset(4)
-	region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/IndexBlockLength + 1
-	return region, nil
-}
-
-func (this *IP2Region) MemorySearch(ipStr string) (ipInfo *IpInfo, err error) {
-	ip, err := ip2long(ipStr)
-	if err != nil {
-		return nil, err
-	}
-
-	h := this.totalBlocks
-	var dataPtr, l int64
-	for l <= h {
-		m := (l + h) >> 1
-		p := this.firstIndexPtr + m*IndexBlockLength
-		sip := this.ipLongAtOffset(p)
-		if ip < sip {
-			h = m - 1
-		} else {
-			eip := this.ipLongAtOffset(p + 4)
-			if ip > eip {
-				l = m + 1
-			} else {
-				dataPtr = this.ipLongAtOffset(p + 8)
-				break
-			}
-		}
-	}
-	if dataPtr == 0 {
-		return nil, nil
-	}
-
-	dataLen := (dataPtr >> 24) & 0xFF
-	dataPtr = dataPtr & 0x00FFFFFF
-	return getIpInfo(this.ipLongAtOffset(dataPtr), this.dbData[(dataPtr)+4:dataPtr+dataLen]), nil
-}
-
-func (this *IP2Region) ipLongAtOffset(offset int64) int64 {
-	return int64(this.dbData[offset]) |
-		int64(this.dbData[offset+1])<<8 |
-		int64(this.dbData[offset+2])<<16 |
-		int64(this.dbData[offset+3])<<24
-}
-
-func ip2long(IpStr string) (int64, error) {
-	bits := strings.Split(IpStr, ".")
-	if len(bits) != 4 {
-		return 0, errors.New("ip format error")
-	}
-
-	var sum int64
-	for i, n := range bits {
-		bit, _ := strconv.ParseInt(n, 10, 64)
-		sum += bit << uint(24-8*i)
-	}
-
-	return sum, nil
-}

internal/iplibrary/ip_list_db.go

@@ -18,7 +18,8 @@ import (
 type IPListDB struct {
 	db *sql.DB
 
-	itemTableName          string
+	itemTableName string
+
 	deleteExpiredItemsStmt *sql.Stmt
 	deleteItemStmt         *sql.Stmt
 	insertItemStmt         *sql.Stmt
@@ -53,7 +54,9 @@ func (this *IPListDB) init() error {
 		remotelogs.Println("IP_LIST_DB", "create data dir '"+this.dir+"'")
 	}
 
-	db, err := sql.Open("sqlite3", "file:"+this.dir+"/ip_list.db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
+	var path = this.dir + "/ip_list.db"
+
+	db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
 	if err != nil {
 		return err
 	}
@@ -66,6 +69,14 @@ func (this *IPListDB) init() error {
 
 	this.db = db
 
+	// 恢复数据库
+	var recoverEnv, _ = os.LookupEnv("EdgeRecover")
+	if len(recoverEnv) > 0 {
+		for _, indexName := range []string{"ip_list_itemId", "ip_list_expiredAt"} {
+			_, _ = db.Exec(`REINDEX "` + indexName + `"`)
+		}
+	}
+
 	// 初始化数据库
 	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.itemTableName + `" (
   "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
@@ -159,6 +170,12 @@ func (this *IPListDB) AddItem(item *pb.IPItem) error {
 	if err != nil {
 		return err
 	}
+
+	// 如果是删除，则不再创建新记录
+	if item.IsDeleted {
+		return nil
+	}
+
 	_, err = this.insertItemStmt.Exec(item.ListId, item.ListType, item.IsGlobal, item.Type, item.Id, item.IpFrom, item.IpTo, item.ExpiredAt, item.EventLevel, item.IsDeleted, item.Version, item.NodeId, item.ServerId)
 	return err
 }
@@ -194,12 +211,12 @@ func (this *IPListDB) ReadMaxVersion() int64 {
 		return 0
 	}
 
-	row := this.selectMaxVersionStmt.QueryRow()
+	var row = this.selectMaxVersionStmt.QueryRow()
 	if row == nil {
 		return 0
 	}
 	var version int64
-	err = row.Scan(&version)
+	err := row.Scan(&version)
 	if err != nil {
 		return 0
 	}

internal/iplibrary/ip_list_db_test.go

@@ -16,6 +16,7 @@ func TestIPListDB_AddItem(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	err = db.AddItem(&pb.IPItem{
 		Id:                            1,
 		IpFrom:                        "192.168.1.101",
@@ -45,6 +46,12 @@ func TestIPListDB_AddItem(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	t.Log("ok")
 }
 

internal/iplibrary/library_interface.go

@@ -1,13 +0,0 @@
-package iplibrary
-
-type LibraryInterface interface {
-	// Load 加载数据库文件
-	Load(dbPath string) error
-
-	// Lookup 查询IP
-	// 返回结果有可能为空
-	Lookup(ip string) (*Result, error)
-
-	// Close 关闭数据库文件
-	Close()
-}

internal/iplibrary/library_ip2region.go

@@ -1,83 +0,0 @@
-package iplibrary
-
-import (
-	"fmt"
-	"github.com/TeaOSLab/EdgeNode/internal/errors"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"net"
-	"strings"
-)
-
-type IP2RegionLibrary struct {
-	db *IP2Region
-}
-
-func (this *IP2RegionLibrary) Load(dbPath string) error {
-	db, err := NewIP2Region(dbPath)
-	if err != nil {
-		return err
-	}
-	this.db = db
-
-	return nil
-}
-
-func (this *IP2RegionLibrary) Lookup(ip string) (*Result, error) {
-	// 暂不支持IPv6
-	if strings.Contains(ip, ":") {
-		return nil, nil
-	}
-	if net.ParseIP(ip) == nil {
-		return nil, nil
-	}
-
-	if this.db == nil {
-		return nil, errors.New("library has not been loaded")
-	}
-
-	defer func() {
-		// 防止panic发生
-		err := recover()
-		if err != nil {
-			remotelogs.Error("IP2RegionLibrary", "panic: "+fmt.Sprintf("%#v", err))
-		}
-	}()
-
-	info, err := this.db.MemorySearch(ip)
-	if err != nil {
-		return nil, err
-	}
-
-	if info == nil {
-		return nil, nil
-	}
-
-	if info.Country == "0" {
-		info.Country = ""
-	}
-	if info.Region == "0" {
-		info.Region = ""
-	}
-	if info.Province == "0" {
-		info.Province = ""
-	}
-	if info.City == "0" {
-		info.City = ""
-	}
-	if info.ISP == "0" {
-		info.ISP = ""
-	}
-
-	return &Result{
-		CityId:   info.CityId,
-		Country:  info.Country,
-		Region:   info.Region,
-		Province: info.Province,
-		City:     info.City,
-		ISP:      info.ISP,
-	}, nil
-}
-
-func (this *IP2RegionLibrary) Close() {
-
-}

internal/iplibrary/library_ip2region_test.go

@@ -1,114 +0,0 @@
-package iplibrary
-
-import (
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/rands"
-	"runtime"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestIP2RegionLibrary_Lookup_MemoryUsage(t *testing.T) {
-	var mem = &runtime.MemStats{}
-	runtime.ReadMemStats(mem)
-
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var mem2 = &runtime.MemStats{}
-	runtime.ReadMemStats(mem2)
-	t.Log((mem2.HeapInuse-mem.HeapInuse)/1024/1024, "MB")
-}
-
-func TestIP2RegionLibrary_Lookup_Single(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, ip := range []string{"8.8.9.9"} {
-		result, err := library.Lookup(ip)
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("IP:", ip, "result:", result)
-	}
-}
-
-func TestIP2RegionLibrary_Lookup(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, ip := range []string{"", "a", "1.1.1", "192.168.1.100", "114.240.223.47", "8.8.9.9", "::1"} {
-		result, err := library.Lookup(ip)
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("IP:", ip, "result:", result)
-	}
-}
-
-func TestIP2RegionLibrary_Lookup_Concurrent(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var count = 4000
-	var wg = sync.WaitGroup{}
-	wg.Add(count)
-	for i := 0; i < count; i++ {
-		go func() {
-			defer wg.Done()
-
-			for i := 0; i < 100; i++ {
-				_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
-			}
-		}()
-	}
-
-	wg.Done()
-	t.Log("ok")
-}
-
-func TestIP2RegionLibrary_Memory(t *testing.T) {
-	library := &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	before := time.Now()
-
-	for i := 0; i < 1_000_000; i++ {
-		_, _ = library.Lookup(strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)) + "." + strconv.Itoa(rands.Int(0, 254)))
-	}
-
-	t.Log("cost:", time.Since(before).Seconds()*1000, "ms")
-}
-
-func BenchmarkIP2RegionLibrary_Lookup(b *testing.B) {
-	runtime.GOMAXPROCS(1)
-
-	var library = &IP2RegionLibrary{}
-	err := library.Load(Tea.Root + "/resources/ipdata/ip2region/ip2region.db")
-	if err != nil {
-		b.Fatal(err)
-	}
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		_, _ = library.Lookup("8.8.8.8")
-	}
-}

internal/iplibrary/manager.go

@@ -1,95 +0,0 @@
-package iplibrary
-
-import (
-	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/TeaOSLab/EdgeNode/internal/errors"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/iwind/TeaGo/Tea"
-	"github.com/iwind/TeaGo/files"
-	"github.com/iwind/TeaGo/types"
-	"regexp"
-	"strings"
-)
-
-var SharedManager = NewManager()
-var SharedLibrary LibraryInterface
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		// 初始化
-		library, err := SharedManager.Load()
-		if err != nil {
-			remotelogs.ErrorObject("IP_LIBRARY", err)
-			return
-		}
-		SharedLibrary = library
-	})
-}
-
-type Manager struct {
-	code string
-}
-
-func NewManager() *Manager {
-	return &Manager{}
-}
-
-func (this *Manager) Load() (LibraryInterface, error) {
-	nodeConfig, err := nodeconfigs.SharedNodeConfig()
-	if err != nil {
-		return nil, err
-	}
-	config := nodeConfig.GlobalConfig
-	if config == nil {
-		config = &serverconfigs.GlobalConfig{}
-	}
-
-	// 当前正在使用的IP库代号
-	code := config.IPLibrary.Code
-	if len(code) == 0 {
-		code = serverconfigs.DefaultIPLibraryType
-	}
-
-	dir := Tea.Root + "/resources/ipdata/" + code
-	var lastVersion int64 = -1
-	lastFilename := ""
-	for _, file := range files.NewFile(dir).List() {
-		filename := file.Name()
-
-		reg := regexp.MustCompile(`^` + regexp.QuoteMeta(code) + `.(\d+)\.`)
-		if reg.MatchString(filename) { // 先查找有版本号的
-			result := reg.FindStringSubmatch(filename)
-			version := types.Int64(result[1])
-			if version > lastVersion {
-				lastVersion = version
-				lastFilename = filename
-			}
-		} else if strings.HasPrefix(filename, code+".") { // 后查找默认的
-			if lastVersion == -1 {
-				lastFilename = filename
-				lastVersion = 0
-			}
-		}
-	}
-
-	if len(lastFilename) == 0 {
-		return nil, errors.New("ip library file not found")
-	}
-
-	var libraryPtr LibraryInterface
-	switch code {
-	case serverconfigs.IPLibraryTypeIP2Region:
-		libraryPtr = &IP2RegionLibrary{}
-	default:
-		return nil, errors.New("invalid ip library code '" + code + "'")
-	}
-
-	err = libraryPtr.Load(dir + "/" + lastFilename)
-	if err != nil {
-		return nil, err
-	}
-
-	return libraryPtr, nil
-}

internal/iplibrary/manager_city.go

@@ -1,155 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/types"
-	"os"
-	"sync"
-	"time"
-)
-
-var SharedCityManager = NewCityManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedCityManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedCityManager.Stop()
-	})
-}
-
-// CityManager 中国省份信息管理
-type CityManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	cityMap  map[string]int64 // provinceName_cityName => cityName
-	dataHash string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewCityManager() *CityManager {
-	return &CityManager{
-		cacheFile: Tea.Root + "/configs/region_city.json.cache",
-		cityMap:   map[string]int64{},
-	}
-}
-
-func (this *CityManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("CITY_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("City_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("CITY_MANAGER", err)
-		}
-	}
-}
-
-func (this *CityManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *CityManager) Lookup(provinceId int64, cityName string) (cityId int64) {
-	this.locker.RLock()
-	cityId, _ = this.cityMap[types.String(provinceId)+"_"+cityName]
-	this.locker.RUnlock()
-	return
-}
-
-// 从缓存中读取
-func (this *CityManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.cityMap = m
-	}
-
-	return nil
-}
-
-// 更新城市信息
-func (this *CityManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionCityRPC().FindAllRegionCities(rpcClient.Context(), &pb.FindAllRegionCitiesRequest{})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, city := range resp.RegionCities {
-		for _, code := range city.Codes {
-			m[types.String(city.RegionProvinceId)+"_"+code] = city.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.cityMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_city_test.go

@@ -1,14 +0,0 @@
-// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package iplibrary
-
-import "testing"
-
-func TestNewCityManager(t *testing.T) {
-	var manager = NewCityManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup(16, "许昌市"))
-}

internal/iplibrary/manager_country.go

@@ -1,153 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"os"
-	"sync"
-	"time"
-)
-
-var SharedCountryManager = NewCountryManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedCountryManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedCountryManager.Stop()
-	})
-}
-
-// CountryManager 国家/地区信息管理
-type CountryManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	countryMap map[string]int64 // countryName => countryId
-	dataHash   string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewCountryManager() *CountryManager {
-	return &CountryManager{
-		cacheFile:  Tea.Root + "/configs/region_country.json.cache",
-		countryMap: map[string]int64{},
-	}
-}
-
-func (this *CountryManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("COUNTRY_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("COUNTRY_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("COUNTRY_MANAGER", err)
-		}
-	}
-}
-
-func (this *CountryManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *CountryManager) Lookup(countryName string) (countryId int64) {
-	this.locker.RLock()
-	countryId, _ = this.countryMap[countryName]
-	this.locker.RUnlock()
-	return countryId
-}
-
-// 从缓存中读取
-func (this *CountryManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.countryMap = m
-	}
-
-	return nil
-}
-
-// 更新国家信息
-func (this *CountryManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionCountryRPC().FindAllRegionCountries(rpcClient.Context(), &pb.FindAllRegionCountriesRequest{})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, country := range resp.RegionCountries {
-		for _, code := range country.Codes {
-			m[code] = country.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.countryMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_country_test.go

@@ -1,57 +0,0 @@
-package iplibrary
-
-import (
-	"runtime"
-	"testing"
-)
-
-func TestCountryManager_load(t *testing.T) {
-	manager := NewCountryManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.countryMap)
-}
-
-func TestCountryManager_loop(t *testing.T) {
-	manager := NewCountryManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.countryMap)
-}
-
-func TestCountryManager_loop_skip(t *testing.T) {
-	manager := NewCountryManager()
-	for i := 0; i < 10; i++ {
-		err := manager.loop()
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestCountryManager_Lookup(t *testing.T) {
-	manager := NewCountryManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup("中国"), manager.Lookup("美国 "))
-}
-
-func BenchmarkCountryManager_Lookup(b *testing.B) {
-	runtime.GOMAXPROCS(1)
-
-	manager := NewCountryManager()
-	err := manager.load()
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	for i := 0; i < b.N; i++ {
-		_ = manager.Lookup("中国")
-	}
-}

internal/iplibrary/manager_ip_list.go

@@ -1,7 +1,9 @@
 package iplibrary
 
 import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
+	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
@@ -18,6 +20,10 @@ var SharedIPListManager = NewIPListManager()
 var IPListUpdateNotify = make(chan bool, 1)
 
 func init() {
+	if teaconst.IsDaemon {
+		return
+	}
+
 	events.On(events.EventLoaded, func() {
 		goman.New(func() {
 			SharedIPListManager.Start()
@@ -26,6 +32,13 @@ func init() {
 	events.On(events.EventQuit, func() {
 		SharedIPListManager.Stop()
 	})
+
+	var ticker = time.NewTicker(24 * time.Hour)
+	goman.New(func() {
+		for range ticker.C {
+			SharedIPListManager.DeleteExpiredItems()
+		}
+	})
 }
 
 // IPListManager IP名单管理
@@ -43,7 +56,7 @@ type IPListManager struct {
 
 func NewIPListManager() *IPListManager {
 	return &IPListManager{
-		pageSize: 500,
+		pageSize: 1000,
 		listMap:  map[int64]*IPList{},
 	}
 }
@@ -111,20 +124,30 @@ func (this *IPListManager) init() {
 		var size int64 = 1000
 		for {
 			items, err := db.ReadItems(offset, size)
+			var l = len(items)
 			if err != nil {
 				remotelogs.Error("IP_LIST_MANAGER", "read ip list from local database failed: "+err.Error())
 			} else {
-				if len(items) == 0 {
+				if l == 0 {
 					break
 				}
 				this.processItems(items, false)
+				if int64(l) < size {
+					break
+				}
 			}
-			offset += int64(len(items))
+			offset += int64(l)
 		}
 	}
 }
 
 func (this *IPListManager) loop() error {
+	// 是否同步IP名单
+	nodeConfig, _ := nodeconfigs.SharedNodeConfig()
+	if nodeConfig != nil && !nodeConfig.EnableIPLists {
+		return nil
+	}
+
 	for {
 		hasNext, err := this.fetch()
 		if err != nil {
@@ -144,7 +167,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
 	if err != nil {
 		return false, err
 	}
-	itemsResp, err := rpcClient.IPItemRPC().ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
+	itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
 		Version: this.version,
 		Size:    this.pageSize,
 	})
@@ -155,7 +178,7 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
 		}
 		return false, err
 	}
-	items := itemsResp.IpItems
+	var items = itemsResp.IpItems
 	if len(items) == 0 {
 		return false, nil
 	}
@@ -182,8 +205,13 @@ func (this *IPListManager) FindList(listId int64) *IPList {
 	return list
 }
 
+func (this *IPListManager) DeleteExpiredItems() {
+	if this.db != nil {
+		_ = this.db.DeleteExpiredItems()
+	}
+}
+
 func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
-	this.locker.Lock()
 	var changedLists = map[*IPList]zero.Zero{}
 	for _, item := range items {
 		var list *IPList
@@ -203,11 +231,15 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 				list = GlobalWhiteIPList
 			}
 		} else { // 其他List
+			this.locker.Lock()
 			list = this.listMap[item.ListId]
+			this.locker.Unlock()
 		}
 		if list == nil {
 			list = NewIPList()
+			this.locker.Lock()
 			this.listMap[item.ListId] = list
+			this.locker.Unlock()
 		}
 
 		changedLists[list] = zero.New()
@@ -246,8 +278,6 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
 		changedList.Sort()
 	}
 
-	this.locker.Unlock()
-
 	if fromRemote {
 		var latestVersion = items[len(items)-1].Version
 		if latestVersion > this.version {

internal/iplibrary/manager_provider.go

@@ -1,154 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"os"
-	"sync"
-	"time"
-)
-
-var SharedProviderManager = NewProviderManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedProviderManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedProviderManager.Stop()
-	})
-}
-
-// ProviderManager 中国省份信息管理
-type ProviderManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	providerMap map[string]int64 // name => id
-	dataHash    string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewProviderManager() *ProviderManager {
-	return &ProviderManager{
-		cacheFile:   Tea.Root + "/configs/region_provider.json.cache",
-		providerMap: map[string]int64{},
-	}
-}
-
-func (this *ProviderManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("PROVIDER_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("PROVIDER_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("PROVIDER_MANAGER", err)
-		}
-	}
-}
-
-func (this *ProviderManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *ProviderManager) Lookup(providerName string) (providerId int64) {
-	this.locker.RLock()
-	providerId, _ = this.providerMap[providerName]
-	this.locker.RUnlock()
-	return
-}
-
-// 从缓存中读取
-func (this *ProviderManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.providerMap = m
-	}
-
-	return nil
-}
-
-// 更新服务商信息
-func (this *ProviderManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionProviderRPC().FindAllRegionProviders(rpcClient.Context(), &pb.FindAllRegionProvidersRequest{})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, provider := range resp.RegionProviders {
-		for _, code := range provider.Codes {
-			m[code] = provider.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.providerMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_provider_test.go

@@ -1,15 +0,0 @@
-// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package iplibrary
-
-import "testing"
-
-func TestNewProviderManager(t *testing.T) {
-	var manager = NewProviderManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup("阿里云"))
-	t.Log(manager.Lookup("阿里云2"))
-}

internal/iplibrary/manager_province.go

@@ -1,160 +0,0 @@
-package iplibrary
-
-import (
-	"crypto/md5"
-	"encoding/json"
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"os"
-	"sync"
-	"time"
-)
-
-const (
-	ChinaCountryId int64 = 1
-)
-
-var SharedProvinceManager = NewProvinceManager()
-
-func init() {
-	events.On(events.EventLoaded, func() {
-		goman.New(func() {
-			SharedProvinceManager.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedProvinceManager.Stop()
-	})
-}
-
-// ProvinceManager 中国省份信息管理
-type ProvinceManager struct {
-	ticker *time.Ticker
-
-	cacheFile string
-
-	provinceMap map[string]int64 // provinceName => provinceId
-	dataHash    string           // 国家JSON的md5
-
-	locker sync.RWMutex
-
-	isUpdated bool
-}
-
-func NewProvinceManager() *ProvinceManager {
-	return &ProvinceManager{
-		cacheFile:   Tea.Root + "/configs/region_province.json.cache",
-		provinceMap: map[string]int64{},
-	}
-}
-
-func (this *ProvinceManager) Start() {
-	// 从缓存中读取
-	err := this.load()
-	if err != nil {
-		remotelogs.ErrorObject("PROVINCE_MANAGER", err)
-	}
-
-	// 第一次更新
-	err = this.loop()
-	if err != nil {
-		remotelogs.ErrorObject("PROVINCE_MANAGER", err)
-	}
-
-	// 定时更新
-	this.ticker = time.NewTicker(4 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("PROVINCE_MANAGER", err)
-		}
-	}
-}
-
-func (this *ProvinceManager) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-func (this *ProvinceManager) Lookup(provinceName string) (provinceId int64) {
-	this.locker.RLock()
-	provinceId, _ = this.provinceMap[provinceName]
-	this.locker.RUnlock()
-	return provinceId
-}
-
-// 从缓存中读取
-func (this *ProvinceManager) load() error {
-	data, err := os.ReadFile(this.cacheFile)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	m := map[string]int64{}
-	err = json.Unmarshal(data, &m)
-	if err != nil {
-		return err
-	}
-	if m != nil && len(m) > 0 {
-		this.provinceMap = m
-	}
-
-	return nil
-}
-
-// 更新省份信息
-func (this *ProvinceManager) loop() error {
-	if this.isUpdated {
-		return nil
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	resp, err := rpcClient.RegionProvinceRPC().FindAllRegionProvincesWithRegionCountryId(rpcClient.Context(), &pb.FindAllRegionProvincesWithRegionCountryIdRequest{
-		RegionCountryId: ChinaCountryId,
-	})
-	if err != nil {
-		return err
-	}
-
-	m := map[string]int64{}
-	for _, province := range resp.RegionProvinces {
-		for _, code := range province.Codes {
-			m[code] = province.Id
-		}
-	}
-
-	// 检查是否有更新
-	data, err := json.Marshal(m)
-	if err != nil {
-		return err
-	}
-	hash := md5.New()
-	hash.Write(data)
-	dataHash := fmt.Sprintf("%x", hash.Sum(nil))
-	if this.dataHash == dataHash {
-		return nil
-	}
-	this.dataHash = dataHash
-
-	this.locker.Lock()
-	this.provinceMap = m
-	this.isUpdated = true
-	this.locker.Unlock()
-
-	// 保存到本地缓存
-
-	err = os.WriteFile(this.cacheFile, data, 0666)
-	return err
-}

internal/iplibrary/manager_province_test.go

@@ -1,57 +0,0 @@
-package iplibrary
-
-import (
-	"runtime"
-	"testing"
-)
-
-func TestProvinceManager_load(t *testing.T) {
-	manager := NewProvinceManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.provinceMap)
-}
-
-func TestProvinceManager_loop(t *testing.T) {
-	manager := NewProvinceManager()
-	err := manager.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok", manager.provinceMap)
-}
-
-func TestProvinceManager_loop_skip(t *testing.T) {
-	manager := NewProvinceManager()
-	for i := 0; i < 10; i++ {
-		err := manager.loop()
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestProvinceManager_Lookup(t *testing.T) {
-	manager := NewProvinceManager()
-	err := manager.load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(manager.Lookup("安徽省"), manager.Lookup("北京市"))
-}
-
-func BenchmarkProvinceManager_Lookup(b *testing.B) {
-	runtime.GOMAXPROCS(1)
-
-	manager := NewProvinceManager()
-	err := manager.load()
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	for i := 0; i < b.N; i++ {
-		_ = manager.Lookup("安徽省")
-	}
-}

internal/iplibrary/manager_test.go

@@ -1,26 +0,0 @@
-package iplibrary
-
-import (
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/dbs"
-	"testing"
-)
-
-func TestManager_Load(t *testing.T) {
-	dbs.NotifyReady()
-
-	manager := NewManager()
-	lib, err := manager.Load()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log(lib.Lookup("1.2.3.4"))
-	t.Log(lib.Lookup("2.3.4.5"))
-	t.Log(lib.Lookup("200.200.200.200"))
-	t.Log(lib.Lookup("202.106.0.20"))
-}
-
-func TestNewManager(t *testing.T) {
-	dbs.NotifyReady()
-	t.Log(SharedLibrary)
-}

internal/iplibrary/updater.go

@@ -1,153 +0,0 @@
-package iplibrary
-
-import (
-	"fmt"
-	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
-	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
-	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
-	"github.com/TeaOSLab/EdgeNode/internal/errors"
-	"github.com/TeaOSLab/EdgeNode/internal/events"
-	"github.com/TeaOSLab/EdgeNode/internal/goman"
-	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
-	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/iwind/TeaGo/Tea"
-	"os"
-	"time"
-)
-
-var SharedUpdater = NewUpdater()
-
-func init() {
-	events.On(events.EventStart, func() {
-		goman.New(func() {
-			SharedUpdater.Start()
-		})
-	})
-	events.On(events.EventQuit, func() {
-		SharedUpdater.Stop()
-	})
-}
-
-// Updater IP库更新程序
-type Updater struct {
-	ticker *time.Ticker
-}
-
-// NewUpdater 获取新对象
-func NewUpdater() *Updater {
-	return &Updater{}
-}
-
-// Start 开始更新
-func (this *Updater) Start() {
-	// 这里不需要太频繁检查更新，因为通常不需要更新IP库
-	this.ticker = time.NewTicker(1 * time.Hour)
-	for range this.ticker.C {
-		err := this.loop()
-		if err != nil {
-			remotelogs.ErrorObject("IP_LIBRARY", err)
-		}
-	}
-}
-
-func (this *Updater) Stop() {
-	if this.ticker != nil {
-		this.ticker.Stop()
-	}
-}
-
-// 单次任务
-func (this *Updater) loop() error {
-	nodeConfig, err := nodeconfigs.SharedNodeConfig()
-	if err != nil {
-		return err
-	}
-	if nodeConfig.GlobalConfig == nil {
-		return nil
-	}
-	code := nodeConfig.GlobalConfig.IPLibrary.Code
-	if len(code) == 0 {
-		code = serverconfigs.DefaultIPLibraryType
-	}
-
-	rpcClient, err := rpc.SharedRPC()
-	if err != nil {
-		return err
-	}
-	libraryResp, err := rpcClient.IPLibraryRPC().FindLatestIPLibraryWithType(rpcClient.Context(), &pb.FindLatestIPLibraryWithTypeRequest{Type: code})
-	if err != nil {
-		return err
-	}
-	lib := libraryResp.IpLibrary
-	if lib == nil || lib.File == nil {
-		return nil
-	}
-
-	typeInfo := serverconfigs.FindIPLibraryWithType(code)
-	if typeInfo == nil {
-		return errors.New("invalid ip library code '" + code + "'")
-	}
-
-	path := Tea.Root + "/resources/ipdata/" + code + "/" + code + "." + fmt.Sprintf("%d", lib.CreatedAt) + typeInfo.GetString("ext")
-
-	// 是否已经存在
-	_, err = os.Stat(path)
-	if err == nil {
-		return nil
-	}
-
-	// 开始下载
-	fileChunkIdsResp, err := rpcClient.FileChunkRPC().FindAllFileChunkIds(rpcClient.Context(), &pb.FindAllFileChunkIdsRequest{FileId: lib.File.Id})
-	if err != nil {
-		return err
-	}
-	chunkIds := fileChunkIdsResp.FileChunkIds
-	if len(chunkIds) == 0 {
-		return nil
-	}
-	isOk := false
-
-	fp, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		// 如果保存不成功就直接删除
-		if !isOk {
-			_ = fp.Close()
-			_ = os.Remove(path)
-		}
-	}()
-	for _, chunkId := range chunkIds {
-		chunkResp, err := rpcClient.FileChunkRPC().DownloadFileChunk(rpcClient.Context(), &pb.DownloadFileChunkRequest{FileChunkId: chunkId})
-		if err != nil {
-			return err
-		}
-		chunk := chunkResp.FileChunk
-
-		if chunk == nil {
-			continue
-		}
-		_, err = fp.Write(chunk.Data)
-		if err != nil {
-			return err
-		}
-	}
-
-	err = fp.Close()
-	if err != nil {
-		return err
-	}
-
-	// 重新加载
-	library, err := SharedManager.Load()
-	if err != nil {
-		return err
-	}
-	SharedLibrary = library
-
-	isOk = true
-
-	return nil
-}

internal/iplibrary/updater_test.go

@@ -1,18 +0,0 @@
-package iplibrary
-
-import (
-	_ "github.com/iwind/TeaGo/bootstrap"
-	"github.com/iwind/TeaGo/dbs"
-	"testing"
-)
-
-func TestUpdater_loop(t *testing.T) {
-	dbs.NotifyReady()
-
-	updater := NewUpdater()
-	err := updater.loop()
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Log("ok")
-}

internal/metrics/manager.go

@@ -4,6 +4,7 @@ package metrics
 
 import (
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/events"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"strconv"
 	"sync"
@@ -11,7 +12,15 @@ import (
 
 var SharedManager = NewManager()
 
+func init() {
+	events.On(events.EventQuit, func() {
+		SharedManager.Quit()
+	})
+}
+
 type Manager struct {
+	isQuiting bool
+
 	tasks         map[int64]*Task    // itemId => *Task
 	categoryTasks map[string][]*Task // category => []*Task
 	locker        sync.RWMutex
@@ -29,6 +38,10 @@ func NewManager() *Manager {
 }
 
 func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
+	if this.isQuiting {
+		return
+	}
+
 	this.locker.Lock()
 	defer this.locker.Unlock()
 
@@ -101,6 +114,10 @@ func (this *Manager) Update(items []*serverconfigs.MetricItemConfig) {
 
 // Add 添加数据
 func (this *Manager) Add(obj MetricInterface) {
+	if this.isQuiting {
+		return
+	}
+
 	this.locker.RLock()
 	for _, task := range this.categoryTasks[obj.MetricCategory()] {
 		task.Add(obj)
@@ -119,3 +136,17 @@ func (this *Manager) HasTCPMetrics() bool {
 func (this *Manager) HasUDPMetrics() bool {
 	return this.hasUDPMetrics
 }
+
+// Quit 退出管理器
+func (this *Manager) Quit() {
+	this.isQuiting = true
+
+	remotelogs.Println("METRIC_MANAGER", "quit")
+
+	this.locker.Lock()
+	for _, task := range this.tasks {
+		_ = task.Stop()
+	}
+	this.tasks = map[int64]*Task{}
+	this.locker.Unlock()
+}

internal/metrics/task.go

@@ -16,6 +16,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
 	"github.com/TeaOSLab/EdgeNode/internal/zero"
 	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/types"
 	_ "github.com/mattn/go-sqlite3"
 	"os"
 	"strconv"
@@ -89,13 +90,23 @@ func (this *Task) Init() error {
 		remotelogs.Println("METRIC", "create data dir '"+dir+"'")
 	}
 
-	db, err := sql.Open("sqlite3", "file:"+dir+"/metric."+strconv.FormatInt(this.item.Id, 10)+".db?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
+	var path = dir + "/metric." + types.String(this.item.Id) + ".db"
+
+	db, err := sql.Open("sqlite3", "file:"+path+"?cache=shared&mode=rwc&_journal_mode=WAL&_sync=OFF")
 	if err != nil {
 		return err
 	}
 	db.SetMaxOpenConns(1)
 	this.db = dbs.NewDB(db)
 
+	// 恢复数据库
+	var recoverEnv, _ = os.LookupEnv("EdgeRecover")
+	if len(recoverEnv) > 0 {
+		for _, indexName := range []string{"serverId", "hash"} {
+			_, _ = db.Exec(`REINDEX "` + indexName + `"`)
+		}
+	}
+
 	if teaconst.EnableDBStat {
 		this.db.EnableStat(true)
 	}
@@ -424,7 +435,7 @@ func (this *Task) Upload(pauseDuration time.Duration) error {
 						return nil, err
 					}
 
-					_, err = rpcClient.MetricStatRPC().UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
+					_, err = rpcClient.MetricStatRPC.UploadMetricStats(rpcClient.Context(), &pb.UploadMetricStatsRequest{
 						MetricStats: pbStats,
 						Time:        currentTime,
 						ServerId:    serverId,

internal/monitor/value_queue.go

@@ -69,7 +69,7 @@ func (this *ValueQueue) Loop() error {
 	}
 
 	for value := range this.valuesChan {
-		_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
+		_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{
 			Item:      value.Item,
 			ValueJSON: value.ValueJSON,
 			CreatedAt: value.CreatedAt,

internal/monitor/value_queue_test.go

@@ -16,9 +16,9 @@ func TestValueQueue_RPC(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = rpcClient.NodeValueRPC().CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
+	_, err = rpcClient.NodeValueRPC.CreateNodeValue(rpcClient.Context(), &pb.CreateNodeValueRequest{})
 	if err != nil {
-		statusErr, ok:= status.FromError(err)
+		statusErr, ok := status.FromError(err)
 		if ok {
 			logs.Println(statusErr.Code())
 		}

internal/nodes/api_stream.go

@@ -1,7 +1,6 @@
 package nodes
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -17,7 +16,7 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
-	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/maps"
 	"net/url"
@@ -73,7 +72,7 @@ func (this *APIStream) loop() error {
 		cancelFunc()
 	}()
 
-	nodeStream, err := rpcClient.NodeRPC().NodeStream(ctx)
+	nodeStream, err := rpcClient.NodeRPC.NodeStream(ctx)
 	if err != nil {
 		if this.isQuiting {
 			return nil
@@ -179,7 +178,7 @@ func (this *APIStream) handleWriteCache(message *pb.NodeStreamMessage) error {
 	}
 
 	expiredAt := time.Now().Unix() + msg.LifeSeconds
-	writer, err := storage.OpenWriter(msg.Key, expiredAt, 200, int64(len(msg.Value)), -1, false)
+	writer, err := storage.OpenWriter(msg.Key, expiredAt, 200, -1, int64(len(msg.Value)), -1, false)
 	if err != nil {
 		this.replyFail(message.RequestId, "prepare writing failed: "+err.Error())
 		return err
@@ -347,15 +346,15 @@ func (this *APIStream) handleCheckSystemdService(message *pb.NodeStreamMessage)
 		return nil
 	}
 
-	var cmd = utils.NewCommandExecutor()
-	shortName := teaconst.SystemdServiceName
-	cmd.Add(systemctl, "is-enabled", shortName)
-	output, err := cmd.Run()
+	var shortName = teaconst.SystemdServiceName
+	var cmd = executils.NewTimeoutCmd(10*time.Second, systemctl, "is-enabled", shortName)
+	cmd.WithStdout()
+	err = cmd.Run()
 	if err != nil {
 		this.replyFail(message.RequestId, "'systemctl' command error: "+err.Error())
 		return nil
 	}
-	if output == "enabled" {
+	if cmd.Stdout() == "enabled" {
 		this.replyOk(message.RequestId, "ok")
 	} else {
 		this.replyFail(message.RequestId, "not installed")
@@ -385,16 +384,15 @@ func (this *APIStream) handleCheckLocalFirewall(message *pb.NodeStreamMessage) e
 			return nil
 		}
 
-		var cmd = exec.Command(nft, "--version")
-		var output = &bytes.Buffer{}
-		cmd.Stdout = output
+		var cmd = executils.NewTimeoutCmd(10*time.Second, nft, "--version")
+		cmd.WithStdout()
 		err = cmd.Run()
 		if err != nil {
 			this.replyFail(message.RequestId, "get version failed: "+err.Error())
 			return nil
 		}
 
-		var outputString = output.String()
+		var outputString = cmd.Stdout()
 		var versionMatches = regexp.MustCompile(`nftables v([\d.]+)`).FindStringSubmatch(outputString)
 		if len(versionMatches) <= 1 {
 			this.replyFail(message.RequestId, "can not get nft version")
@@ -409,7 +407,7 @@ func (this *APIStream) handleCheckLocalFirewall(message *pb.NodeStreamMessage) e
 		var protectionConfig = sharedNodeConfig.DDoSProtection
 		err = firewalls.SharedDDoSProtectionManager.Apply(protectionConfig)
 		if err != nil {
-			this.replyFail(message.RequestId, dataMessage.Name+"was installed, but apply DDoS protection config failed: "+err.Error())
+			this.replyFail(message.RequestId, dataMessage.Name+" was installed, but apply DDoS protection config failed: "+err.Error())
 		} else {
 			this.replyOk(message.RequestId, string(result.AsJSON()))
 		}

internal/nodes/client_conn.go

@@ -3,60 +3,96 @@
 package nodes
 
 import (
+	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
+	"github.com/TeaOSLab/EdgeNode/internal/conns"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
 	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/stats"
 	"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"github.com/TeaOSLab/EdgeNode/internal/waf"
+	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/types"
 	"net"
 	"os"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 )
 
 // ClientConn 客户端连接
 type ClientConn struct {
-	once sync.Once
+	BaseClientConn
 
-	isTLS       bool
-	hasDeadline bool
-	hasRead     bool
+	createdAt int64
 
-	isLO bool // 是否为环路
+	isTLS   bool
+	isHTTP  bool
+	hasRead bool
+
+	isLO          bool // 是否为环路
+	isInAllowList bool
 
 	hasResetSYNFlood bool
 
-	BaseClientConn
+	lastReadAt  int64
+	lastWriteAt int64
+	lastErr     error
+
+	readDeadlineTime int64
+	isShortReading   bool // reading header or tls handshake
+
+	isDebugging      bool
+	autoReadTimeout  bool
+	autoWriteTimeout bool
 }
 
-func NewClientConn(conn net.Conn, isTLS bool, quickClose bool) net.Conn {
-	if quickClose {
-		// TCP
-		tcpConn, ok := conn.(*net.TCPConn)
-		if ok {
-			// TODO 可以在配置中设置此值
-			_ = tcpConn.SetLinger(nodeconfigs.DefaultTCPLinger)
-		}
-	}
-
+func NewClientConn(rawConn net.Conn, isHTTP bool, isTLS bool, isInAllowList bool) net.Conn {
 	// 是否为环路
-	var remoteAddr = conn.RemoteAddr().String()
+	var remoteAddr = rawConn.RemoteAddr().String()
 	var isLO = strings.HasPrefix(remoteAddr, "127.0.0.1:") || strings.HasPrefix(remoteAddr, "[::1]:")
 
-	return &ClientConn{
-		BaseClientConn: BaseClientConn{rawConn: conn},
+	var conn = &ClientConn{
+		BaseClientConn: BaseClientConn{rawConn: rawConn},
 		isTLS:          isTLS,
+		isHTTP:         isHTTP,
 		isLO:           isLO,
+		isInAllowList:  isInAllowList,
+		createdAt:      time.Now().Unix(),
 	}
+
+	var globalServerConfig = sharedNodeConfig.GlobalServerConfig
+	if globalServerConfig != nil {
+		var performanceConfig = globalServerConfig.Performance
+		conn.isDebugging = performanceConfig.Debug
+		conn.autoReadTimeout = performanceConfig.AutoReadTimeout
+		conn.autoWriteTimeout = performanceConfig.AutoWriteTimeout
+	}
+
+	if isHTTP {
+		// TODO 可以在配置中设置此值
+		_ = conn.SetLinger(nodeconfigs.DefaultTCPLinger)
+	}
+
+	// 加入到Map
+	conns.SharedMap.Add(conn)
+
+	return conn
 }
 
 func (this *ClientConn) Read(b []byte) (n int, err error) {
+	if this.isDebugging {
+		this.lastReadAt = time.Now().Unix()
+
+		defer func() {
+			if err != nil {
+				this.lastErr = errors.New("read error: " + err.Error())
+			}
+		}()
+	}
+
 	// 环路直接读取
 	if this.isLO {
 		n, err = this.rawConn.Read(b)
@@ -66,40 +102,39 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
 		return
 	}
 
-	// TLS
-	if this.isTLS {
-		if !this.hasDeadline {
-			_ = this.rawConn.SetReadDeadline(time.Now().Add(time.Duration(nodeconfigs.DefaultTLSHandshakeTimeout) * time.Second)) // TODO 握手超时时间可以设置
-			this.hasDeadline = true
-			defer func() {
-				_ = this.rawConn.SetReadDeadline(time.Time{})
-			}()
-		}
+	// 设置读超时时间
+	if this.isHTTP && !this.isWebsocket && !this.isShortReading && this.autoReadTimeout {
+		this.setHTTPReadTimeout()
 	}
 
 	// 开始读取
 	n, err = this.rawConn.Read(b)
 	if n > 0 {
 		atomic.AddUint64(&teaconst.InTrafficBytes, uint64(n))
-		if !this.hasRead {
-			this.hasRead = true
-		}
+		this.hasRead = true
 	}
 
-	var isHandshakeError = err != nil && os.IsTimeout(err) && !this.hasRead
-	if isHandshakeError {
+	// 检测是否为超时错误
+	var isTimeout = err != nil && os.IsTimeout(err)
+	var isHandshakeError = isTimeout && !this.hasRead
+	if isTimeout {
 		_ = this.SetLinger(0)
+	} else {
+		_ = this.SetLinger(nodeconfigs.DefaultTCPLinger)
 	}
 
-	// SYN Flood检测
-	if this.serverId == 0 || !this.hasResetSYNFlood {
-		var synFloodConfig = sharedNodeConfig.SYNFloodConfig()
-		if synFloodConfig != nil && synFloodConfig.IsOn {
-			if isHandshakeError {
-				this.increaseSYNFlood(synFloodConfig)
-			} else if err == nil && !this.hasResetSYNFlood {
-				this.hasResetSYNFlood = true
-				this.resetSYNFlood()
+	// 忽略白名单和局域网
+	if this.isHTTP && !this.isInAllowList && !utils.IsLocalIP(this.RawIP()) {
+		// SYN Flood检测
+		if this.serverId == 0 || !this.hasResetSYNFlood {
+			var synFloodConfig = sharedNodeConfig.SYNFloodConfig()
+			if synFloodConfig != nil && synFloodConfig.IsOn {
+				if isHandshakeError {
+					this.increaseSYNFlood(synFloodConfig)
+				} else if err == nil && !this.hasResetSYNFlood {
+					this.hasResetSYNFlood = true
+					this.resetSYNFlood()
+				}
 			}
 		}
 	}
@@ -108,18 +143,54 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
 }
 
 func (this *ClientConn) Write(b []byte) (n int, err error) {
+	if this.isDebugging {
+		this.lastWriteAt = time.Now().Unix()
+
+		defer func() {
+			if err != nil {
+				this.lastErr = errors.New("write error: " + err.Error())
+			}
+		}()
+	}
+
+	// 设置写超时时间
+	if this.autoWriteTimeout {
+		// TODO L2 -> L1 写入时不限制时间
+		var timeoutSeconds = len(b) / 1024
+		if timeoutSeconds < 3 {
+			timeoutSeconds = 3
+		}
+		_ = this.rawConn.SetWriteDeadline(time.Now().Add(time.Duration(timeoutSeconds) * time.Second)) // TODO 时间可以设置
+	}
+
+	// 延长读超时时间
+	if this.isHTTP && !this.isWebsocket && this.autoReadTimeout {
+		this.setHTTPReadTimeout()
+	}
+
+	// 开始写入
 	n, err = this.rawConn.Write(b)
 	if n > 0 {
-		atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
-
 		// 统计当前服务带宽
 		if this.serverId > 0 {
-			if !this.isLO { // 环路不统计带宽，避免缓存预热等行为产生带宽
+			if !this.isLO || Tea.IsTesting() { // 环路不统计带宽，避免缓存预热等行为产生带宽
+				atomic.AddUint64(&teaconst.OutTrafficBytes, uint64(n))
 				stats.SharedBandwidthStatManager.Add(this.userId, this.serverId, int64(n))
 			}
 		}
 	}
 
+	// 如果是写入超时，则立即关闭连接
+	if err != nil && os.IsTimeout(err) {
+		// TODO 考虑对多次慢连接的IP做出惩罚
+		conn, ok := this.rawConn.(LingerConn)
+		if ok {
+			_ = conn.SetLinger(0)
+		}
+
+		_ = this.Close()
+	}
+
 	return
 }
 
@@ -132,6 +203,9 @@ func (this *ClientConn) Close() error {
 	// 不能加条件限制，因为服务配置随时有变化
 	sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
 
+	// 从conn map中移除
+	conns.SharedMap.Remove(this)
+
 	return err
 }
 
@@ -148,6 +222,26 @@ func (this *ClientConn) SetDeadline(t time.Time) error {
 }
 
 func (this *ClientConn) SetReadDeadline(t time.Time) error {
+	// 如果开启了HTTP自动读超时选项，则自动控制超时时间
+	if this.isHTTP && !this.isWebsocket && this.autoReadTimeout {
+		this.isShortReading = false
+
+		var unixTime = t.Unix()
+		if unixTime < 10 {
+			return nil
+		}
+		if unixTime == this.readDeadlineTime {
+			return nil
+		}
+		this.readDeadlineTime = unixTime
+		var seconds = -time.Since(t)
+		if seconds <= 0 || seconds > HTTPIdleTimeout {
+			return nil
+		}
+		if seconds < HTTPIdleTimeout-1*time.Second {
+			this.isShortReading = true
+		}
+	}
 	return this.rawConn.SetReadDeadline(t)
 }
 
@@ -155,6 +249,22 @@ func (this *ClientConn) SetWriteDeadline(t time.Time) error {
 	return this.rawConn.SetWriteDeadline(t)
 }
 
+func (this *ClientConn) CreatedAt() int64 {
+	return this.createdAt
+}
+
+func (this *ClientConn) LastReadAt() int64 {
+	return this.lastReadAt
+}
+
+func (this *ClientConn) LastWriteAt() int64 {
+	return this.lastWriteAt
+}
+
+func (this *ClientConn) LastErr() error {
+	return this.lastErr
+}
+
 func (this *ClientConn) resetSYNFlood() {
 	ttlcache.SharedCache.Delete("SYN_FLOOD:" + this.RawIP())
 }
@@ -177,7 +287,17 @@ func (this *ClientConn) increaseSYNFlood(synFloodConfig *firewallconfigs.SYNFloo
 			if timeout <= 0 {
 				timeout = 600
 			}
+
+			// 关闭当前连接
+			_ = this.SetLinger(0)
+			_ = this.Close()
+
 			waf.SharedIPBlackList.RecordIP(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip, time.Now().Unix()+int64(timeout), 0, true, 0, 0, "疑似SYN Flood攻击，当前1分钟"+types.String(result)+"次空连接")
 		}
 	}
 }
+
+// 设置读超时时间
+func (this *ClientConn) setHTTPReadTimeout() {
+	_ = this.SetReadDeadline(time.Now().Add(HTTPIdleTimeout))
+}

internal/nodes/client_conn_base.go

@@ -2,7 +2,10 @@
 
 package nodes
 
-import "net"
+import (
+	"crypto/tls"
+	"net"
+)
 
 type BaseClientConn struct {
 	rawConn net.Conn
@@ -13,7 +16,11 @@ type BaseClientConn struct {
 	remoteAddr string
 	hasLimit   bool
 
+	isWebsocket bool
+
 	isClosed bool
+
+	rawIP string
 }
 
 func (this *BaseClientConn) IsClosed() bool {
@@ -42,6 +49,17 @@ func (this *BaseClientConn) Bind(serverId int64, remoteAddr string, maxConnsPerS
 // SetServerId 设置服务ID
 func (this *BaseClientConn) SetServerId(serverId int64) {
 	this.serverId = serverId
+
+	// 设置包装前连接
+	switch conn := this.rawConn.(type) {
+	case *tls.Conn:
+		nativeConn, ok := conn.NetConn().(ClientConnInterface)
+		if ok {
+			nativeConn.SetServerId(serverId)
+		}
+	case *ClientConn:
+		conn.SetServerId(serverId)
+	}
 }
 
 // ServerId 读取当前连接绑定的服务ID
@@ -52,6 +70,17 @@ func (this *BaseClientConn) ServerId() int64 {
 // SetUserId 设置所属服务的用户ID
 func (this *BaseClientConn) SetUserId(userId int64) {
 	this.userId = userId
+
+	// 设置包装前连接
+	switch conn := this.rawConn.(type) {
+	case *tls.Conn:
+		nativeConn, ok := conn.NetConn().(ClientConnInterface)
+		if ok {
+			nativeConn.SetUserId(userId)
+		}
+	case *ClientConn:
+		conn.SetUserId(userId)
+	}
 }
 
 // UserId 获取当前连接所属服务的用户ID
@@ -61,14 +90,30 @@ func (this *BaseClientConn) UserId() int64 {
 
 // RawIP 原本IP
 func (this *BaseClientConn) RawIP() string {
+	if len(this.rawIP) > 0 {
+		return this.rawIP
+	}
+
 	ip, _, _ := net.SplitHostPort(this.rawConn.RemoteAddr().String())
+	this.rawIP = ip
 	return ip
 }
 
 // TCPConn 转换为TCPConn
-func (this *BaseClientConn) TCPConn() (*net.TCPConn, bool) {
-	conn, ok := this.rawConn.(*net.TCPConn)
-	return conn, ok
+func (this *BaseClientConn) TCPConn() (tcpConn *net.TCPConn, ok bool) {
+	// 设置包装前连接
+	switch conn := this.rawConn.(type) {
+	case *tls.Conn:
+		var internalConn = conn.NetConn()
+		clientConn, ok := internalConn.(*ClientConn)
+		if ok {
+			return clientConn.TCPConn()
+		}
+		tcpConn, ok = internalConn.(*net.TCPConn)
+	default:
+		tcpConn, ok = this.rawConn.(*net.TCPConn)
+	}
+	return
 }
 
 // SetLinger 设置Linger
@@ -79,3 +124,7 @@ func (this *BaseClientConn) SetLinger(seconds int) error {
 	}
 	return nil
 }
+
+func (this *BaseClientConn) SetIsWebsocket(isWebsocket bool) {
+	this.isWebsocket = isWebsocket
+}

internal/nodes/client_conn_interface.go

@@ -23,4 +23,7 @@ type ClientConnInterface interface {
 
 	// UserId 获取当前连接所属服务的用户ID
 	UserId() int64
+
+	// SetIsWebsocket 设置是否为Websocket
+	SetIsWebsocket(isWebsocket bool)
 }

internal/nodes/client_listener.go

@@ -8,19 +8,20 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/waf"
 	"net"
+	"time"
 )
 
 // ClientListener 客户端网络监听
 type ClientListener struct {
 	rawListener net.Listener
+	isHTTP      bool
 	isTLS       bool
-	quickClose  bool
 }
 
-func NewClientListener(listener net.Listener, quickClose bool) *ClientListener {
+func NewClientListener(listener net.Listener, isHTTP bool) *ClientListener {
 	return &ClientListener{
 		rawListener: listener,
-		quickClose:  quickClose,
+		isHTTP:      isHTTP,
 	}
 }
 
@@ -40,12 +41,32 @@ func (this *ClientListener) Accept() (net.Conn, error) {
 
 	// 是否在WAF名单中
 	ip, _, err := net.SplitHostPort(conn.RemoteAddr().String())
+	var isInAllowList = false
 	if err == nil {
-		canGoNext, _ := iplibrary.AllowIP(ip, 0)
-		var beingDenied = !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) &&
-			waf.SharedIPBlackList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
+		canGoNext, inAllowList := iplibrary.AllowIP(ip, 0)
+		isInAllowList = inAllowList
+		if !waf.SharedIPWhiteList.Contains(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip) {
+			expiresAt, ok := waf.SharedIPBlackList.ContainsExpires(waf.IPTypeAll, firewallconfigs.FirewallScopeGlobal, 0, ip)
+			if ok {
+				var timeout = expiresAt - time.Now().Unix()
+				if timeout > 0 {
+					canGoNext = false
 
-		if !canGoNext || beingDenied {
+					if timeout > 3600 {
+						timeout = 3600
+					}
+
+					// 使用本地防火墙延长封禁
+					var fw = firewalls.Firewall()
+					if fw != nil && !fw.IsMock() {
+						// 这里 int(int64) 转换的前提是限制了 timeout <= 3600，否则将有整型溢出的风险
+						_ = fw.DropSourceIP(ip, int(timeout), true)
+					}
+				}
+			}
+		}
+
+		if !canGoNext {
 			tcpConn, ok := conn.(*net.TCPConn)
 			if ok {
 				_ = tcpConn.SetLinger(0)
@@ -53,19 +74,11 @@ func (this *ClientListener) Accept() (net.Conn, error) {
 
 			_ = conn.Close()
 
-			// 使用本地防火墙延长封禁
-			if beingDenied {
-				var fw = firewalls.Firewall()
-				if fw != nil && !fw.IsMock() {
-					_ = fw.DropSourceIP(ip, 120, true)
-				}
-			}
-
 			return this.Accept()
 		}
 	}
 
-	return NewClientConn(conn, this.isTLS, this.quickClose), nil
+	return NewClientConn(conn, this.isHTTP, this.isTLS, isInAllowList), nil
 }
 
 func (this *ClientListener) Close() error {

internal/nodes/http_access_log_queue.go

@@ -7,6 +7,8 @@ import (
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"strings"
 	"time"
 )
@@ -41,7 +43,11 @@ func (this *HTTPAccessLogQueue) Start() {
 	for range ticker.C {
 		err := this.loop()
 		if err != nil {
-			remotelogs.Error("ACCESS_LOG_QUEUE", err.Error())
+			if rpc.IsConnError(err) {
+				remotelogs.Debug("ACCESS_LOG_QUEUE", err.Error())
+			} else {
+				remotelogs.Error("ACCESS_LOG_QUEUE", err.Error())
+			}
 		}
 	}
 }
@@ -96,7 +102,7 @@ Loop:
 		this.rpcClient = client
 	}
 
-	_, err := this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
+	_, err := this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
 	if err != nil {
 		// 是否包含了invalid UTF-8
 		if strings.Contains(err.Error(), "string field contains invalid UTF-8") {
@@ -105,7 +111,20 @@ Loop:
 			}
 
 			// 重新提交
-			_, err = this.rpcClient.HTTPAccessLogRPC().CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
+			_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
+			return err
+		}
+
+		// 是否请求内容过大
+		statusCode, ok := status.FromError(err)
+		if ok && statusCode.Code() == codes.ResourceExhausted {
+			// 去除Body
+			for _, accessLog := range accessLogs {
+				accessLog.RequestBody = nil
+			}
+
+			// 重新提交
+			_, err = this.rpcClient.HTTPAccessLogRPC.CreateHTTPAccessLogs(this.rpcClient.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: accessLogs})
 			return err
 		}
 

internal/nodes/http_access_log_queue_test.go

@@ -55,7 +55,7 @@ func TestHTTPAccessLogQueue_Push(t *testing.T) {
 	//	logs.PrintAsJSON(accessLog)
 
 	//t.Log(strings.ToValidUTF8(string(utf8Bytes), ""))
-	_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
+	_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
 		accessLog,
 	}})
 	if err != nil {
@@ -99,7 +99,7 @@ func TestHTTPAccessLogQueue_Push2(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = client.HTTPAccessLogRPC().CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
+	_, err = client.HTTPAccessLogRPC.CreateHTTPAccessLogs(client.Context(), &pb.CreateHTTPAccessLogsRequest{HttpAccessLogs: []*pb.HTTPAccessLog{
 		accessLog,
 	}})
 	if err != nil {

internal/nodes/http_cache_task_manager.go

@@ -81,7 +81,7 @@ func (this *HTTPCacheTaskManager) Start() {
 
 		if rpcClient != nil {
 			for taskReq := range this.taskQueue {
-				_, err := rpcClient.ServerRPC().PurgeServerCache(rpcClient.Context(), taskReq)
+				_, err := rpcClient.ServerRPC.PurgeServerCache(rpcClient.Context(), taskReq)
 				if err != nil {
 					remotelogs.Error("HTTP_CACHE_TASK_MANAGER", "create purge task failed: "+err.Error())
 				}
@@ -104,8 +104,12 @@ func (this *HTTPCacheTaskManager) Loop() error {
 		return err
 	}
 
-	resp, err := rpcClient.HTTPCacheTaskKeyRPC().FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
+	resp, err := rpcClient.HTTPCacheTaskKeyRPC.FindDoingHTTPCacheTaskKeys(rpcClient.Context(), &pb.FindDoingHTTPCacheTaskKeysRequest{})
 	if err != nil {
+		// 忽略连接错误
+		if rpc.IsConnError(err) {
+			return nil
+		}
 		return err
 	}
 
@@ -131,7 +135,7 @@ func (this *HTTPCacheTaskManager) Loop() error {
 		pbResults = append(pbResults, pbResult)
 	}
 
-	_, err = rpcClient.HTTPCacheTaskKeyRPC().UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
+	_, err = rpcClient.HTTPCacheTaskKeyRPC.UpdateHTTPCacheTaskKeysStatus(rpcClient.Context(), &pb.UpdateHTTPCacheTaskKeysStatusRequest{KeyResults: pbResults})
 	if err != nil {
 		return err
 	}

internal/nodes/http_client_pool.go

@@ -95,11 +95,11 @@ func (this *HTTPClientPool) Client(req *HTTPRequest,
 		numberCPU = 8
 	}
 	if maxConnections <= 0 {
-		maxConnections = numberCPU * 32
+		maxConnections = numberCPU * 64
 	}
 
 	if idleConns <= 0 {
-		idleConns = numberCPU * 8
+		idleConns = numberCPU * 16
 	}
 
 	// 可以判断为Ln节点请求

internal/nodes/http_request.go

@@ -6,10 +6,10 @@ import (
 	"errors"
 	"fmt"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	iplib "github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
-	"github.com/TeaOSLab/EdgeNode/internal/iplibrary"
 	"github.com/TeaOSLab/EdgeNode/internal/metrics"
 	"github.com/TeaOSLab/EdgeNode/internal/stats"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
@@ -68,6 +68,7 @@ type HTTPRequest struct {
 	filePath             string                            // 请求的文件名，仅在读取Root目录下的内容时不为空
 	origin               *serverconfigs.OriginConfig       // 源站
 	originAddr           string                            // 源站实际地址
+	originStatus         int32                             // 源站响应代码
 	errors               []string                          // 错误信息
 	rewriteRule          *serverconfigs.HTTPRewriteRule    // 匹配到的重写规则
 	rewriteReplace       string                            // 重写规则的目标
@@ -228,6 +229,22 @@ func (this *HTTPRequest) Do() {
 			}
 		}
 
+		// 防盗链
+		if !this.isSubRequest && this.web.Referers != nil && this.web.Referers.IsOn {
+			if this.doCheckReferers() {
+				this.doEnd()
+				return
+			}
+		}
+
+		// UA名单
+		if !this.isSubRequest && this.web.UserAgent != nil && this.web.UserAgent.IsOn {
+			if this.doCheckUserAgent() {
+				this.doEnd()
+				return
+			}
+		}
+
 		// 访问控制
 		if !this.isSubRequest && this.web.Auth != nil && this.web.Auth.IsOn {
 			if this.doAuth() {
@@ -512,6 +529,16 @@ func (this *HTTPRequest) configureWeb(web *serverconfigs.HTTPWebConfig, isTop bo
 		this.web.Auth = web.Auth
 	}
 
+	// referers
+	if web.Referers != nil && (web.Referers.IsPrior || isTop) {
+		this.web.Referers = web.Referers
+	}
+
+	// user agent
+	if web.UserAgent != nil && (web.UserAgent.IsPrior || isTop) {
+		this.web.UserAgent = web.UserAgent
+	}
+
 	// request limit
 	if web.RequestLimit != nil && (web.RequestLimit.IsPrior || isTop) {
 		this.web.RequestLimit = web.RequestLimit
@@ -677,7 +704,7 @@ func (this *HTTPRequest) Format(source string) string {
 		case "remoteAddrValue":
 			return this.requestRemoteAddr(false)
 		case "rawRemoteAddr":
-			addr := this.RawReq.RemoteAddr
+			var addr = this.RawReq.RemoteAddr
 			host, _, err := net.SplitHostPort(addr)
 			if err == nil {
 				addr = host
@@ -744,6 +771,8 @@ func (this *HTTPRequest) Format(source string) string {
 			return strconv.FormatInt(this.requestFromTime.Unix(), 10)
 		case "host":
 			return this.ReqHost
+		case "cname":
+			return this.ReqServer.CNameDomain
 		case "referer":
 			return this.RawReq.Referer()
 		case "referer.host":
@@ -928,42 +957,47 @@ func (this *HTTPRequest) Format(source string) string {
 
 		// geo
 		if prefix == "geo" {
-			result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
+			var result = iplib.LookupIP(this.requestRemoteAddr(true))
 
 			switch suffix {
 			case "country.name":
-				if result != nil {
-					return result.Country
+				if result != nil && result.IsOk() {
+					return result.CountryName()
 				}
 				return ""
 			case "country.id":
-				if result != nil {
-					return types.String(iplibrary.SharedCountryManager.Lookup(result.Country))
+				if result != nil && result.IsOk() {
+					return types.String(result.CountryId())
 				}
 				return "0"
 			case "province.name":
-				if result != nil {
-					return result.Province
+				if result != nil && result.IsOk() {
+					return result.ProvinceName()
 				}
 				return ""
 			case "province.id":
-				if result != nil {
-					return types.String(iplibrary.SharedProvinceManager.Lookup(result.Province))
+				if result != nil && result.IsOk() {
+					return types.String(result.ProvinceId())
 				}
 				return "0"
 			case "city.name":
-				if result != nil {
-					return result.City
+				if result != nil && result.IsOk() {
+					return result.CityName()
 				}
 				return ""
 			case "city.id":
-				if result != nil {
-					var provinceId = iplibrary.SharedProvinceManager.Lookup(result.Province)
-					if provinceId > 0 {
-						return types.String(iplibrary.SharedCityManager.Lookup(provinceId, result.City))
-					} else {
-						return "0"
-					}
+				if result != nil && result.IsOk() {
+					return types.String(result.CityId())
+				}
+				return "0"
+			case "town.name":
+				if result != nil && result.IsOk() {
+					return result.TownName()
+				}
+				return ""
+			case "town.id":
+				if result != nil && result.IsOk() {
+					return types.String(result.TownId())
 				}
 				return "0"
 			}
@@ -971,16 +1005,16 @@ func (this *HTTPRequest) Format(source string) string {
 
 		// ips
 		if prefix == "isp" {
-			result, _ := iplibrary.SharedLibrary.Lookup(this.requestRemoteAddr(true))
+			var result = iplib.LookupIP(this.requestRemoteAddr(true))
 
 			switch suffix {
 			case "name":
-				if result != nil {
-					return result.ISP
+				if result != nil && result.IsOk() {
+					return result.ProviderName()
 				}
 			case "id":
-				if result != nil {
-					return types.String(iplibrary.SharedProviderManager.Lookup(result.ISP))
+				if result != nil && result.IsOk() {
+					return types.String(result.ProviderId())
 				}
 				return "0"
 			}
@@ -1098,7 +1132,7 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
 	}
 
 	// Remote-Addr
-	remoteAddr := this.RawReq.RemoteAddr
+	var remoteAddr = this.RawReq.RemoteAddr
 	host, _, err := net.SplitHostPort(remoteAddr)
 	if err == nil {
 		if supportVar {
@@ -1112,6 +1146,8 @@ func (this *HTTPRequest) requestRemoteAddr(supportVar bool) string {
 
 // 获取请求的客户端地址列表
 func (this *HTTPRequest) requestRemoteAddrs() (result []string) {
+	result = append(result, this.requestRemoteAddr(true))
+
 	// X-Forwarded-For
 	var forwardedFor = this.RawReq.Header.Get("X-Forwarded-For")
 	if len(forwardedFor) > 0 {
@@ -1167,7 +1203,7 @@ func (this *HTTPRequest) requestRemoteUser() string {
 
 // Path 请求的URL中路径部分
 func (this *HTTPRequest) Path() string {
-	uri, err := url.ParseRequestURI(this.rawURI)
+	uri, err := url.ParseRequestURI(this.uri)
 	if err != nil {
 		return ""
 	}
@@ -1315,7 +1351,7 @@ func (this *HTTPRequest) RemoteAddr() string {
 }
 
 func (this *HTTPRequest) RawRemoteAddr() string {
-	addr := this.RawReq.RemoteAddr
+	var addr = this.RawReq.RemoteAddr
 	host, _, err := net.SplitHostPort(addr)
 	if err == nil {
 		addr = host
@@ -1423,18 +1459,21 @@ func (this *HTTPRequest) Done() {
 func (this *HTTPRequest) Close() {
 	this.Done()
 
-	requestConn := this.RawReq.Context().Value(HTTPConnContextKey)
+	var requestConn = this.RawReq.Context().Value(HTTPConnContextKey)
 	if requestConn == nil {
 		return
 	}
 
+	lingerConn, ok := requestConn.(LingerConn)
+	if ok {
+		_ = lingerConn.SetLinger(0)
+	}
+
 	conn, ok := requestConn.(net.Conn)
 	if ok {
 		_ = conn.Close()
 		return
 	}
-
-	return
 }
 
 // Allow 放行
@@ -1530,7 +1569,7 @@ func (this *HTTPRequest) processRequestHeaders(reqHeader http.Header) {
 			}
 
 			// 是否已删除
-			if this.web.ResponseHeaderPolicy.ContainsDeletedHeader(header.Name) {
+			if this.web.RequestHeaderPolicy.ContainsDeletedHeader(header.Name) {
 				continue
 			}
 
@@ -1581,17 +1620,31 @@ func (this *HTTPRequest) fixRequestHeader(header http.Header) {
 			header.Del(k)
 			k = strings.ReplaceAll(k, "-Websocket-", "-WebSocket-")
 			header[k] = v
-		} else if k == "Www-Authenticate" {
+		} else if strings.HasPrefix(k, "Sec-Ch") {
 			header.Del(k)
-			header["WWW-Authenticate"] = v
+			k = strings.ReplaceAll(k, "Sec-Ch-Ua", "Sec-CH-UA")
+			header[k] = v
+		} else {
+			switch k {
+			case "Www-Authenticate":
+				header.Del(k)
+				header["WWW-Authenticate"] = v
+			case "A-Im":
+				header.Del(k)
+				header["A-IM"] = v
+			case "Content-Md5":
+				header.Del(k)
+				header["Content-MD5"] = v
+			case "Sec-Gpc":
+				header.Del(k)
+				header["Content-GPC"] = v
+			}
 		}
 	}
 }
 
 // 处理自定义Response Header
-func (this *HTTPRequest) processResponseHeaders(statusCode int) {
-	var responseHeader = this.writer.Header()
-
+func (this *HTTPRequest) processResponseHeaders(responseHeader http.Header, statusCode int) {
 	// 删除/添加/替换Header
 	// TODO 实现AddTrailers
 	if this.web.ResponseHeaderPolicy != nil && this.web.ResponseHeaderPolicy.IsOn {
@@ -1654,6 +1707,36 @@ func (this *HTTPRequest) processResponseHeaders(statusCode int) {
 				responseHeader[header.Name] = []string{headerValue}
 			}
 		}
+
+		// CORS
+		if this.web.ResponseHeaderPolicy.CORS != nil && this.web.ResponseHeaderPolicy.CORS.IsOn {
+			var corsConfig = this.web.ResponseHeaderPolicy.CORS
+
+			// Allow-Origin
+			if len(corsConfig.AllowOrigin) == 0 {
+				var origin = this.RawReq.Header.Get("Origin")
+				if len(origin) > 0 {
+					responseHeader.Set("Access-Control-Allow-Origin", origin)
+				}
+			} else {
+				responseHeader.Set("Access-Control-Allow-Origin", corsConfig.AllowOrigin)
+			}
+
+			// Allow-Methods
+			if len(corsConfig.AllowMethods) == 0 {
+				responseHeader.Set("Access-Control-Allow-Methods", "PUT, GET, POST, DELETE, HEAD, OPTIONS")
+			} else {
+				responseHeader.Set("Access-Control-Allow-Methods", strings.Join(corsConfig.AllowMethods, ", "))
+			}
+
+			// Max-Age
+			if corsConfig.MaxAge > 0 {
+				responseHeader.Set("Access-Control-Max-Age", types.String(corsConfig.MaxAge))
+			}
+
+			// Allow-Credentials
+			responseHeader.Set("Access-Control-Allow-Credentials", "true")
+		}
 	}
 
 	// HSTS
@@ -1685,10 +1768,10 @@ func (this *HTTPRequest) bytePool(contentLength int64) *utils.BytePool {
 		return utils.BytePool1k
 	}
 	if contentLength < 32768 { // 32K
-		return utils.BytePool4k
+		return utils.BytePool16k
 	}
 	if contentLength < 131072 { // 128K
-		return utils.BytePool16k
+		return utils.BytePool32k
 	}
 	return utils.BytePool32k
 }
@@ -1721,7 +1804,7 @@ func (this *HTTPRequest) canIgnore(err error) bool {
 	}
 
 	// HTTP/2流错误
-	if err.Error() == "http2: stream closed" || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
+	if err.Error() == "http2: stream closed" || strings.Contains(err.Error(), "stream error") || err.Error() == "client disconnected" { // errStreamClosed, errClientDisconnected
 		return true
 	}
 

internal/nodes/http_request_acme.go

@@ -21,7 +21,7 @@ func (this *HTTPRequest) doACME() (shouldStop bool) {
 		return false
 	}
 
-	keyResp, err := rpcClient.ACMEAuthenticationRPC().FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
+	keyResp, err := rpcClient.ACMEAuthenticationRPC.FindACMEAuthenticationKeyWithToken(rpcClient.Context(), &pb.FindACMEAuthenticationKeyWithTokenRequest{Token: token})
 	if err != nil {
 		remotelogs.Error("RPC", "[ACME]read key for token failed: "+err.Error())
 		return false

internal/nodes/http_request_auth.go

@@ -19,7 +19,10 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 		if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
 			continue
 		}
-		b, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
+		if !ref.AuthPolicy.MatchRequest(this.RawReq) {
+			continue
+		}
+		ok, newURI, uriChanged, err := ref.AuthPolicy.Filter(this.RawReq, func(subReq *http.Request) (status int, err error) {
 			subReq.TLS = this.RawReq.TLS
 			subReq.RemoteAddr = this.RawReq.RemoteAddr
 			subReq.Host = this.RawReq.Host
@@ -36,24 +39,32 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
 			this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
 			return
 		}
-		if b {
+		if ok {
+			if uriChanged {
+				this.uri = newURI
+			}
+			this.tags = append(this.tags, ref.AuthPolicy.Type)
 			return
 		} else {
+			// Basic Auth比较特殊
 			if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
-				var method = ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
-				var headerValue = "Basic realm=\""
-				if len(method.Realm) > 0 {
-					headerValue += method.Realm
-				} else {
-					headerValue += this.ReqHost
+				method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
+				if ok {
+					var headerValue = "Basic realm=\""
+					if len(method.Realm) > 0 {
+						headerValue += method.Realm
+					} else {
+						headerValue += this.ReqHost
+					}
+					headerValue += "\""
+					if len(method.Charset) > 0 {
+						headerValue += ", charset=\"" + method.Charset + "\""
+					}
+					this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
 				}
-				headerValue += "\""
-				if len(method.Charset) > 0 {
-					headerValue += ", charset=\"" + method.Charset + "\""
-				}
-				this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
 			}
 			this.writer.WriteHeader(http.StatusUnauthorized)
+			this.tags = append(this.tags, ref.AuthPolicy.Type)
 			return true
 		}
 	}

internal/nodes/http_request_cache.go

@@ -19,6 +19,11 @@ import (
 
 // 读取缓存
 func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
+	// 需要动态Upgrade的不缓存
+	if len(this.RawReq.Header.Get("Upgrade")) > 0 {
+		return
+	}
+
 	this.cacheCanTryStale = false
 
 	var cachePolicy = this.ReqServer.HTTPCachePolicy
@@ -44,12 +49,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	// 检查服务独立的缓存条件
 	refType := ""
 	for _, cacheRef := range this.web.Cache.CacheRefs {
-		if !cacheRef.IsOn ||
-			cacheRef.Conds == nil ||
-			!cacheRef.Conds.HasRequestConds() {
+		if !cacheRef.IsOn {
 			continue
 		}
-		if cacheRef.Conds.MatchRequest(this.Format) {
+		if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
+			(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
 			if cacheRef.IsReverse {
 				return
 			}
@@ -61,12 +65,11 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	if this.cacheRef == nil && !this.web.Cache.DisablePolicyRefs {
 		// 检查策略默认的缓存条件
 		for _, cacheRef := range cachePolicy.CacheRefs {
-			if !cacheRef.IsOn ||
-				cacheRef.Conds == nil ||
-				!cacheRef.Conds.HasRequestConds() {
+			if !cacheRef.IsOn {
 				continue
 			}
-			if cacheRef.Conds.MatchRequest(this.Format) {
+			if (cacheRef.Conds != nil && cacheRef.Conds.HasRequestConds() && cacheRef.Conds.MatchRequest(this.Format)) ||
+				(cacheRef.SimpleCond != nil && cacheRef.SimpleCond.Match(this.Format)) {
 				if cacheRef.IsReverse {
 					return
 				}
@@ -157,7 +160,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		for _, subKey := range subKeys {
 			err := storage.Delete(subKey)
 			if err != nil {
-				remotelogs.Error("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
+				remotelogs.ErrorServer("HTTP_REQUEST_CACHE", "purge failed: "+err.Error())
 			}
 		}
 
@@ -262,7 +265,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 			}
 
 			if !this.canIgnore(err) {
-				remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: open cache failed: "+err.Error())
+				remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: open cache failed: "+err.Error())
 			}
 			return
 		}
@@ -292,37 +295,34 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		}
 	}
 
-	var pool = this.bytePool(fileSize)
-	var buf = pool.Get()
-	defer func() {
-		pool.Put(buf)
-	}()
-
 	// 读取Header
-	var headerBuf = []byte{}
+	var headerData = []byte{}
 	this.writer.SetSentHeaderBytes(reader.HeaderSize())
-	err = reader.ReadHeader(buf, func(n int) (goNext bool, err error) {
-		headerBuf = append(headerBuf, buf[:n]...)
+	var headerPool = this.bytePool(reader.HeaderSize())
+	var headerBuf = headerPool.Get()
+	err = reader.ReadHeader(headerBuf, func(n int) (goNext bool, err error) {
+		headerData = append(headerData, headerBuf[:n]...)
 		for {
-			nIndex := bytes.Index(headerBuf, []byte{'\n'})
+			nIndex := bytes.Index(headerData, []byte{'\n'})
 			if nIndex >= 0 {
-				row := headerBuf[:nIndex]
+				row := headerData[:nIndex]
 				spaceIndex := bytes.Index(row, []byte{':'})
 				if spaceIndex <= 0 {
 					return false, errors.New("invalid header '" + string(row) + "'")
 				}
 
 				this.writer.Header().Set(string(row[:spaceIndex]), string(row[spaceIndex+1:]))
-				headerBuf = headerBuf[nIndex+1:]
+				headerData = headerData[nIndex+1:]
 			} else {
 				break
 			}
 		}
 		return true, nil
 	})
+	headerPool.Put(headerBuf)
 	if err != nil {
 		if !this.canIgnore(err) {
-			remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read header failed: "+err.Error())
+			remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read header failed: "+err.Error())
 		}
 		return
 	}
@@ -374,7 +374,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	// 支持 If-None-Match
 	if !this.isLnRequest && !isPartialCache && len(eTag) > 0 && this.requestHeader("If-None-Match") == eTag {
 		// 自定义Header
-		this.processResponseHeaders(http.StatusNotModified)
+		this.processResponseHeaders(this.writer.Header(), http.StatusNotModified)
 		this.addExpiresHeader(reader.ExpiresAt())
 		this.writer.WriteHeader(http.StatusNotModified)
 		this.isCached = true
@@ -386,7 +386,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 	// 支持 If-Modified-Since
 	if !this.isLnRequest && !isPartialCache && len(modifiedTime) > 0 && this.requestHeader("If-Modified-Since") == modifiedTime {
 		// 自定义Header
-		this.processResponseHeaders(http.StatusNotModified)
+		this.processResponseHeaders(this.writer.Header(), http.StatusNotModified)
 		this.addExpiresHeader(reader.ExpiresAt())
 		this.writer.WriteHeader(http.StatusNotModified)
 		this.isCached = true
@@ -395,7 +395,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		return true
 	}
 
-	this.processResponseHeaders(reader.Status())
+	this.processResponseHeaders(this.writer.Header(), reader.Status())
 	this.addExpiresHeader(reader.ExpiresAt())
 
 	// 返回上级节点过期时间
@@ -424,7 +424,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 		if supportRange {
 			if len(rangeHeader) > 0 {
 				if fileSize == 0 {
-					this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
+					this.processResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
 					this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 					return true
 				}
@@ -432,7 +432,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 				if len(ranges) == 0 {
 					ranges, ok = httpRequestParseRangeHeader(rangeHeader)
 					if !ok {
-						this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
+						this.processResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
 						this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 						return true
 					}
@@ -441,7 +441,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 					for k, r := range ranges {
 						r2, ok := r.Convert(fileSize)
 						if !ok {
-							this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
+							this.processResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
 							this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 							return true
 						}
@@ -457,23 +457,26 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 			respHeader.Set("Content-Length", strconv.FormatInt(ranges[0].Length(), 10))
 			this.writer.WriteHeader(http.StatusPartialContent)
 
-			err = reader.ReadBodyRange(buf, ranges[0].Start(), ranges[0].End(), func(n int) (goNext bool, err error) {
-				_, err = this.writer.Write(buf[:n])
+			var pool = this.bytePool(fileSize)
+			var bodyBuf = pool.Get()
+			err = reader.ReadBodyRange(bodyBuf, ranges[0].Start(), ranges[0].End(), func(n int) (goNext bool, err error) {
+				_, err = this.writer.Write(bodyBuf[:n])
 				if err != nil {
 					return false, errWritingToClient
 				}
 				return true, nil
 			})
+			pool.Put(bodyBuf)
 			if err != nil {
 				this.varMapping["cache.status"] = "MISS"
 
 				if err == caches.ErrInvalidRange {
-					this.processResponseHeaders(http.StatusRequestedRangeNotSatisfiable)
+					this.processResponseHeaders(this.writer.Header(), http.StatusRequestedRangeNotSatisfiable)
 					this.writer.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 					return true
 				}
 				if !this.canIgnore(err) {
-					remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
+					remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
 				}
 				return
 			}
@@ -510,16 +513,19 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 					}
 				}
 
-				err := reader.ReadBodyRange(buf, r.Start(), r.End(), func(n int) (goNext bool, err error) {
-					_, err = this.writer.Write(buf[:n])
+				var pool = this.bytePool(fileSize)
+				var bodyBuf = pool.Get()
+				err := reader.ReadBodyRange(bodyBuf, r.Start(), r.End(), func(n int) (goNext bool, err error) {
+					_, err = this.writer.Write(bodyBuf[:n])
 					if err != nil {
 						return false, errWritingToClient
 					}
 					return true, nil
 				})
+				pool.Put(bodyBuf)
 				if err != nil {
 					if !this.canIgnore(err) {
-						remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
+						remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: "+err.Error())
 					}
 					return true
 				}
@@ -540,15 +546,18 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 			this.writer.Prepare(resp, fileSize, reader.Status(), false)
 			this.writer.WriteHeader(reader.Status())
 
+			var pool = this.bytePool(fileSize)
+			var bodyBuf = pool.Get()
 			if storage.CanSendfile() {
 				if fp, canSendFile := this.writer.canSendfile(); canSendFile {
-					this.writer.sentBodyBytes, err = io.CopyBuffer(this.writer.rawWriter, fp, buf)
+					this.writer.sentBodyBytes, err = io.CopyBuffer(this.writer.rawWriter, fp, bodyBuf)
 				} else {
-					_, err = io.CopyBuffer(this.writer, resp.Body, buf)
+					_, err = io.CopyBuffer(this.writer, resp.Body, bodyBuf)
 				}
 			} else {
-				_, err = io.CopyBuffer(this.writer, resp.Body, buf)
+				_, err = io.CopyBuffer(this.writer, resp.Body, bodyBuf)
 			}
+			pool.Put(bodyBuf)
 			if err == io.EOF {
 				err = nil
 			}
@@ -556,7 +565,7 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 				this.varMapping["cache.status"] = "MISS"
 
 				if !this.canIgnore(err) {
-					remotelogs.Warn("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read body failed: "+err.Error())
+					remotelogs.WarnServer("HTTP_REQUEST_CACHE", this.URL()+": read from cache failed: read body failed: "+err.Error())
 				}
 				return
 			}
@@ -619,7 +628,14 @@ func (this *HTTPRequest) tryPartialReader(storage caches.StorageInterface, key s
 	}()
 
 	// 检查范围
+	//const maxFirstSpan = 16 << 20 // TODO 可以在缓存策略中设置此值
 	for index, r := range ranges {
+		// 没有指定结束位置时，自动指定一个
+		/**if r.Start() >= 0 && r.End() == -1 {
+			if partialReader.MaxLength() > r.Start()+maxFirstSpan {
+				r[1] = r.Start() + maxFirstSpan
+			}
+		}**/
 		r1, ok := r.Convert(partialReader.MaxLength())
 		if !ok {
 			return nil, nil

internal/nodes/http_request_error.go

@@ -25,10 +25,10 @@ const httpStatusPageTemplate = `<!DOCTYPE html>
 </html>`
 
 func (this *HTTPRequest) write404() {
-	this.writeCode(http.StatusNotFound)
+	this.writeCode(http.StatusNotFound, "", "")
 }
 
-func (this *HTTPRequest) writeCode(statusCode int) {
+func (this *HTTPRequest) writeCode(statusCode int, enMessage string, zhMessage string) {
 	if this.doPage(statusCode) {
 		return
 	}
@@ -42,12 +42,22 @@ func (this *HTTPRequest) writeCode(statusCode int) {
 		case "requestId":
 			return this.requestId
 		case "message":
-			return "" // 空
+			var acceptLanguages = this.RawReq.Header.Get("Accept-Language")
+			if len(acceptLanguages) > 0 {
+				var index = strings.Index(acceptLanguages, ",")
+				if index > 0 {
+					var firstLanguage = acceptLanguages[:index]
+					if firstLanguage == "zh-CN" {
+						return zhMessage
+					}
+				}
+			}
+			return enMessage
 		}
 		return "${" + varName + "}"
 	})
 
-	this.processResponseHeaders(statusCode)
+	this.processResponseHeaders(this.writer.Header(), statusCode)
 	this.writer.WriteHeader(statusCode)
 
 	_, _ = this.writer.Write([]byte(pageContent))
@@ -100,7 +110,7 @@ func (this *HTTPRequest) write50x(err error, statusCode int, enMessage string, z
 		return "${" + varName + "}"
 	})
 
-	this.processResponseHeaders(statusCode)
+	this.processResponseHeaders(this.writer.Header(), statusCode)
 	this.writer.WriteHeader(statusCode)
 
 	_, _ = this.writer.Write([]byte(pageContent))

internal/nodes/http_request_fastcgi.go

@@ -187,7 +187,7 @@ func (this *HTTPRequest) doFastcgi() (shouldStop bool) {
 
 	// 响应Header
 	this.writer.AddHeaders(resp.Header)
-	this.processResponseHeaders(resp.StatusCode)
+	this.processResponseHeaders(this.writer.Header(), resp.StatusCode)
 
 	// 准备
 	this.writer.Prepare(resp, resp.ContentLength, resp.StatusCode, true)

internal/nodes/http_request_host_redirect.go

@@ -1,7 +1,11 @@
 package nodes
 
 import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"github.com/iwind/TeaGo/types"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -13,7 +17,7 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 	if this.web.MergeSlashes {
 		urlPath = utils.CleanPath(urlPath)
 	}
-	fullURL := this.requestScheme() + "://" + this.ReqHost + urlPath
+	var fullURL = this.requestScheme() + "://" + this.ReqHost + urlPath
 	for _, u := range this.web.HostRedirects {
 		if !u.IsOn {
 			continue
@@ -21,11 +25,50 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 		if !u.MatchRequest(this.Format) {
 			continue
 		}
-		if u.MatchPrefix { // 匹配前缀
-			if strings.HasPrefix(fullURL, u.BeforeURL) {
-				afterURL := u.AfterURL
-				if u.KeepRequestURI {
-					afterURL += this.RawReq.URL.RequestURI()
+		if len(u.Type) == 0 || u.Type == serverconfigs.HTTPHostRedirectTypeURL {
+			if u.MatchPrefix { // 匹配前缀
+				if strings.HasPrefix(fullURL, u.BeforeURL) {
+					afterURL := u.AfterURL
+					if u.KeepRequestURI {
+						afterURL += this.RawReq.URL.RequestURI()
+					}
+
+					// 前后是否一致
+					if fullURL == afterURL {
+						return false
+					}
+
+					if u.Status <= 0 {
+						u.Status = http.StatusTemporaryRedirect
+					}
+					this.processResponseHeaders(this.writer.Header(), u.Status)
+					http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
+					return true
+				}
+			} else if u.MatchRegexp { // 正则匹配
+				var reg = u.BeforeURLRegexp()
+				if reg == nil {
+					continue
+				}
+				var matches = reg.FindStringSubmatch(fullURL)
+				if len(matches) == 0 {
+					continue
+				}
+				var afterURL = u.AfterURL
+				for i, match := range matches {
+					afterURL = strings.ReplaceAll(afterURL, "${"+strconv.Itoa(i)+"}", match)
+				}
+
+				var subNames = reg.SubexpNames()
+				if len(subNames) > 0 {
+					for _, subName := range subNames {
+						if len(subName) > 0 {
+							index := reg.SubexpIndex(subName)
+							if index > -1 {
+								afterURL = strings.ReplaceAll(afterURL, "${"+subName+"}", matches[index])
+							}
+						}
+					}
 				}
 
 				// 前后是否一致
@@ -33,69 +76,6 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 					return false
 				}
 
-				if u.Status <= 0 {
-					this.processResponseHeaders(http.StatusTemporaryRedirect)
-					http.Redirect(this.RawWriter, this.RawReq, afterURL, http.StatusTemporaryRedirect)
-				} else {
-					this.processResponseHeaders(u.Status)
-					http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
-				}
-				return true
-			}
-		} else if u.MatchRegexp { // 正则匹配
-			reg := u.BeforeURLRegexp()
-			if reg == nil {
-				continue
-			}
-			matches := reg.FindStringSubmatch(fullURL)
-			if len(matches) == 0 {
-				continue
-			}
-			afterURL := u.AfterURL
-			for i, match := range matches {
-				afterURL = strings.ReplaceAll(afterURL, "${"+strconv.Itoa(i)+"}", match)
-			}
-
-			subNames := reg.SubexpNames()
-			if len(subNames) > 0 {
-				for _, subName := range subNames {
-					if len(subName) > 0 {
-						index := reg.SubexpIndex(subName)
-						if index > -1 {
-							afterURL = strings.ReplaceAll(afterURL, "${"+subName+"}", matches[index])
-						}
-					}
-				}
-			}
-
-			// 前后是否一致
-			if fullURL == afterURL {
-				return false
-			}
-
-			if u.KeepArgs {
-				var qIndex = strings.Index(this.uri, "?")
-				if qIndex >= 0 {
-					afterURL += this.uri[qIndex:]
-				}
-			}
-
-			if u.Status <= 0 {
-				this.processResponseHeaders(http.StatusTemporaryRedirect)
-				http.Redirect(this.RawWriter, this.RawReq, afterURL, http.StatusTemporaryRedirect)
-			} else {
-				this.processResponseHeaders(u.Status)
-				http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
-			}
-			return true
-		} else { // 精准匹配
-			if fullURL == u.RealBeforeURL() {
-				// 前后是否一致
-				if fullURL == u.AfterURL {
-					return false
-				}
-
-				var afterURL = u.AfterURL
 				if u.KeepArgs {
 					var qIndex = strings.Index(this.uri, "?")
 					if qIndex >= 0 {
@@ -104,12 +84,127 @@ func (this *HTTPRequest) doHostRedirect() (blocked bool) {
 				}
 
 				if u.Status <= 0 {
-					this.processResponseHeaders(http.StatusTemporaryRedirect)
-					http.Redirect(this.RawWriter, this.RawReq, afterURL, http.StatusTemporaryRedirect)
-				} else {
-					this.processResponseHeaders(u.Status)
-					http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
+					u.Status = http.StatusTemporaryRedirect
 				}
+				this.processResponseHeaders(this.writer.Header(), u.Status)
+				http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
+				return true
+			} else { // 精准匹配
+				if fullURL == u.RealBeforeURL() {
+					// 前后是否一致
+					if fullURL == u.AfterURL {
+						return false
+					}
+
+					var afterURL = u.AfterURL
+					if u.KeepArgs {
+						var qIndex = strings.Index(this.uri, "?")
+						if qIndex >= 0 {
+							afterURL += this.uri[qIndex:]
+						}
+					}
+
+					if u.Status <= 0 {
+						u.Status = http.StatusTemporaryRedirect
+					}
+					this.processResponseHeaders(this.writer.Header(), u.Status)
+					http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
+					return true
+				}
+			}
+		} else if u.Type == serverconfigs.HTTPHostRedirectTypeDomain {
+			if len(u.DomainAfter) == 0 {
+				continue
+			}
+
+			var reqHost = this.ReqHost
+
+			// 忽略跳转前端口
+			if u.DomainBeforeIgnorePorts {
+				h, _, err := net.SplitHostPort(reqHost)
+				if err == nil && len(h) > 0 {
+					reqHost = h
+				}
+			}
+
+			// 如果跳转前后域名一致，则终止
+			if u.DomainAfter == reqHost {
+				return false
+			}
+
+			var scheme = u.DomainAfterScheme
+			if len(scheme) == 0 {
+				scheme = this.requestScheme()
+			}
+			if u.DomainsAll || configutils.MatchDomains(u.DomainsBefore, reqHost) {
+				var afterURL = scheme + "://" + u.DomainAfter + urlPath
+				if fullURL == afterURL {
+					// 终止匹配
+					return false
+				}
+				if u.Status <= 0 {
+					u.Status = http.StatusTemporaryRedirect
+				}
+				this.processResponseHeaders(this.writer.Header(), u.Status)
+
+				// 参数
+				var qIndex = strings.Index(this.uri, "?")
+				if qIndex >= 0 {
+					afterURL += this.uri[qIndex:]
+				}
+
+				http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
+				return true
+			}
+		} else if u.Type == serverconfigs.HTTPHostRedirectTypePort {
+			if u.PortAfter <= 0 {
+				continue
+			}
+
+			var scheme = u.PortAfterScheme
+			if len(scheme) == 0 {
+				scheme = this.requestScheme()
+			}
+
+			reqHost, reqPort, _ := net.SplitHostPort(this.ReqHost)
+			if len(reqHost) == 0 {
+				reqHost = this.ReqHost
+			}
+			if len(reqPort) == 0 {
+				switch this.requestScheme() {
+				case "http":
+					reqPort = "80"
+				case "https":
+					reqPort = "443"
+				}
+			}
+
+			// 如果跳转前后端口一致，则终止
+			if reqPort == types.String(u.PortAfter) {
+				return false
+			}
+
+			var containsPort = false
+			if u.PortsAll {
+				containsPort = true
+			} else {
+				containsPort = u.ContainsPort(types.Int(reqPort))
+			}
+			if containsPort {
+				var newReqHost = reqHost
+				if !((scheme == "http" && u.PortAfter == 80) || (scheme == "https" && u.PortAfter == 443)) {
+					newReqHost += ":" + types.String(u.PortAfter)
+				}
+				var afterURL = scheme + "://" + newReqHost + urlPath
+				if fullURL == afterURL {
+					// 终止匹配
+					return false
+				}
+				if u.Status <= 0 {
+					u.Status = http.StatusTemporaryRedirect
+				}
+				this.processResponseHeaders(this.writer.Header(), u.Status)
+				http.Redirect(this.RawWriter, this.RawReq, afterURL, u.Status)
 				return true
 			}
 		}

internal/nodes/http_request_limit.go

@@ -18,7 +18,7 @@ func (this *HTTPRequest) doRequestLimit() (shouldStop bool) {
 	// TODO 处理分片提交的内容
 	if this.web.RequestLimit.MaxBodyBytes() > 0 &&
 		this.RawReq.ContentLength > this.web.RequestLimit.MaxBodyBytes() {
-		this.writeCode(http.StatusRequestEntityTooLarge)
+		this.writeCode(http.StatusRequestEntityTooLarge, "", "")
 		return true
 	}
 
@@ -29,7 +29,7 @@ func (this *HTTPRequest) doRequestLimit() (shouldStop bool) {
 			clientConn, ok := requestConn.(ClientConnInterface)
 			if ok && !clientConn.IsBound() {
 				if !clientConn.Bind(this.ReqServer.Id, this.requestRemoteAddr(true), this.web.RequestLimit.MaxConns, this.web.RequestLimit.MaxConnsPerIP) {
-					this.writeCode(http.StatusTooManyRequests)
+					this.writeCode(http.StatusTooManyRequests, "", "")
 					this.Close()
 					return true
 				}

internal/nodes/http_request_log.go

@@ -51,27 +51,46 @@ func (this *HTTPRequest) log() {
 		addr = addr[:index]
 	}
 
+	var serverGlobalConfig = this.nodeConfig.GlobalServerConfig
+
 	// 请求Cookie
 	var cookies = map[string]string{}
-	if ref == nil || ref.ContainsField(serverconfigs.HTTPAccessLogFieldCookie) {
-		for _, cookie := range this.RawReq.Cookies() {
-			cookies[cookie.Name] = cookie.Value
+	var enableCookies = false
+	if serverGlobalConfig == nil || serverGlobalConfig.HTTPAccessLog.EnableCookies {
+		enableCookies = true
+		if ref == nil || ref.ContainsField(serverconfigs.HTTPAccessLogFieldCookie) {
+			for _, cookie := range this.RawReq.Cookies() {
+				cookies[cookie.Name] = cookie.Value
+			}
 		}
 	}
 
 	// 请求Header
 	var pbReqHeader = map[string]*pb.Strings{}
-	if ref == nil || ref.ContainsField(serverconfigs.HTTPAccessLogFieldHeader) {
-		for k, v := range this.RawReq.Header {
-			pbReqHeader[k] = &pb.Strings{Values: v}
+	if serverGlobalConfig == nil || serverGlobalConfig.HTTPAccessLog.EnableRequestHeaders {
+		if ref == nil || ref.ContainsField(serverconfigs.HTTPAccessLogFieldHeader) {
+			// 是否只记录通用Header
+			var commonHeadersOnly = serverGlobalConfig != nil && serverGlobalConfig.HTTPAccessLog.CommonRequestHeadersOnly
+
+			for k, v := range this.RawReq.Header {
+				if commonHeadersOnly && !serverconfigs.IsCommonRequestHeader(k) {
+					continue
+				}
+				if !enableCookies && k == "Cookie" {
+					continue
+				}
+				pbReqHeader[k] = &pb.Strings{Values: v}
+			}
 		}
 	}
 
 	// 响应Header
 	var pbResHeader = map[string]*pb.Strings{}
-	if ref == nil || ref.ContainsField(serverconfigs.HTTPAccessLogFieldSentHeader) {
-		for k, v := range this.writer.Header() {
-			pbResHeader[k] = &pb.Strings{Values: v}
+	if serverGlobalConfig == nil || serverGlobalConfig.HTTPAccessLog.EnableResponseHeaders {
+		if ref == nil || ref.ContainsField(serverconfigs.HTTPAccessLogFieldSentHeader) {
+			for k, v := range this.writer.Header() {
+				pbResHeader[k] = &pb.Strings{Values: v}
+			}
 		}
 	}
 
@@ -146,6 +165,7 @@ func (this *HTTPRequest) log() {
 	if this.origin != nil {
 		accessLog.OriginId = this.origin.Id
 		accessLog.OriginAddress = this.originAddr
+		accessLog.OriginStatus = this.originStatus
 	}
 
 	// 请求Body

internal/nodes/http_request_metrics.go

@@ -34,17 +34,7 @@ func (this *HTTPRequest) MetricValue(value string) (result int64, ok bool) {
 		}
 		return this.RawReq.ContentLength + hl, true
 	case "${countConnection}":
-		metricNewConnMapLocker.Lock()
-		_, ok := metricNewConnMap[this.RawReq.RemoteAddr]
-		if ok {
-			delete(metricNewConnMap, this.RawReq.RemoteAddr)
-		}
-		metricNewConnMapLocker.Unlock()
-		if ok {
-			return 1, true
-		} else {
-			return 0, false
-		}
+		return 1, true
 	}
 	return 0, false
 }

internal/nodes/http_request_mismatch.go

@@ -32,7 +32,7 @@ func (this *HTTPRequest) doMismatch() {
 	}
 
 	// 根据配置进行相应的处理
-	if sharedNodeConfig.GlobalConfig != nil && sharedNodeConfig.GlobalConfig.HTTPAll.MatchDomainStrictly {
+	if sharedNodeConfig.GlobalServerConfig != nil && sharedNodeConfig.GlobalServerConfig.HTTPAll.MatchDomainStrictly {
 		// 检查cc
 		// TODO 可以在管理端配置是否开启以及最多尝试次数
 		if len(remoteIP) > 0 {
@@ -46,7 +46,7 @@ func (this *HTTPRequest) doMismatch() {
 		}
 
 		// 处理当前连接
-		var httpAllConfig = sharedNodeConfig.GlobalConfig.HTTPAll
+		var httpAllConfig = sharedNodeConfig.GlobalServerConfig.HTTPAll
 		var mismatchAction = httpAllConfig.DomainMismatchAction
 		if mismatchAction != nil && mismatchAction.Code == "page" {
 			if mismatchAction.Options != nil {

internal/nodes/http_request_page.go

@@ -60,11 +60,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
 					// 修改状态码
 					if page.NewStatus > 0 {
 						// 自定义响应Headers
-						this.processResponseHeaders(page.NewStatus)
+						this.processResponseHeaders(this.writer.Header(), page.NewStatus)
 						this.writer.Prepare(nil, stat.Size(), page.NewStatus, true)
 						this.writer.WriteHeader(page.NewStatus)
 					} else {
-						this.processResponseHeaders(status)
+						this.processResponseHeaders(this.writer.Header(), status)
 						this.writer.Prepare(nil, stat.Size(), status, true)
 						this.writer.WriteHeader(status)
 					}
@@ -99,11 +99,11 @@ func (this *HTTPRequest) doPage(status int) (shouldStop bool) {
 				// 修改状态码
 				if page.NewStatus > 0 {
 					// 自定义响应Headers
-					this.processResponseHeaders(page.NewStatus)
+					this.processResponseHeaders(this.writer.Header(), page.NewStatus)
 					this.writer.Prepare(nil, int64(len(content)), page.NewStatus, true)
 					this.writer.WriteHeader(page.NewStatus)
 				} else {
-					this.processResponseHeaders(status)
+					this.processResponseHeaders(this.writer.Header(), status)
 					this.writer.Prepare(nil, int64(len(content)), status, true)
 					this.writer.WriteHeader(status)
 				}