ChenKaiLiuG/GoEdgeNode
Archived
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

优化IP名单同步速度

Browse Source
This commit is contained in:
刘祥超
2023-04-19 12:01:02 +08:00
parent 4dc25fb71e
commit 37ddff86f1
20 changed files with 334 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/iplibrary/ip_list_db.go
+92 -25
View File
@@ -17,13 +17,17 @@ import (
type IPListDB struct {
db *dbs.DB
itemTableName string
itemTableName string
versionTableName string
deleteExpiredItemsStmt *dbs.Stmt
deleteItemStmt *dbs.Stmt
insertItemStmt *dbs.Stmt
selectItemsStmt *dbs.Stmt
selectMaxVersionStmt *dbs.Stmt
deleteExpiredItemsStmt *dbs.Stmt
deleteItemStmt *dbs.Stmt
insertItemStmt *dbs.Stmt
selectItemsStmt *dbs.Stmt
selectMaxItemVersionStmt *dbs.Stmt
selectVersionStmt *dbs.Stmt
updateVersionStmt *dbs.Stmt
cleanTicker *time.Ticker
@@ -34,9 +38,10 @@ type IPListDB struct {
func NewIPListDB() (*IPListDB, error) {
var db = &IPListDB{
itemTableName: "ipItems",
dir: filepath.Clean(Tea.Root + "/data"),
cleanTicker: time.NewTicker(24 * time.Hour),
itemTableName: "ipItems",
versionTableName: "versions",
dir: filepath.Clean(Tea.Root + "/data"),
cleanTicker: time.NewTicker(24 * time.Hour),
}
err := db.init()
return db, err
@@ -108,6 +113,15 @@ ON "` + this.itemTableName + `" (
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS "` + this.versionTableName + `" (
"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"version" integer DEFAULT 0
);
`)
if err != nil {
return err
}
// 初始化SQL语句
this.deleteExpiredItemsStmt, err = this.db.Prepare(`DELETE FROM "` + this.itemTableName + `" WHERE "expiredAt">0 AND "expiredAt"<?`)
if err != nil {
@@ -129,7 +143,20 @@ ON "` + this.itemTableName + `" (
return err
}
this.selectMaxVersionStmt, err = this.db.Prepare(`SELECT "version" FROM "` + this.itemTableName + `" ORDER BY "id" DESC LIMIT 1`)
this.selectMaxItemVersionStmt, err = this.db.Prepare(`SELECT "version" FROM "` + this.itemTableName + `" ORDER BY "id" DESC LIMIT 1`)
if err != nil {
return err
}
this.selectVersionStmt, err = this.db.Prepare(`SELECT "version" FROM "` + this.versionTableName + `" LIMIT 1`)
if err != nil {
return err
}
this.updateVersionStmt, err = this.db.Prepare(`REPLACE INTO "` + this.versionTableName + `" ("id", "version") VALUES (1, ?)`)
if err != nil {
return err
}
this.db = db
@@ -172,11 +199,15 @@ func (this *IPListDB) AddItem(item *pb.IPItem) error {
// 如果是删除，则不再创建新记录
if item.IsDeleted {
return nil
return this.UpdateMaxVersion(item.Version)
}
_, err = this.insertItemStmt.Exec(item.ListId, item.ListType, item.IsGlobal, item.Type, item.Id, item.IpFrom, item.IpTo, item.ExpiredAt, item.EventLevel, item.IsDeleted, item.Version, item.NodeId, item.ServerId)
return err
if err != nil {
return err
}
return this.UpdateMaxVersion(item.Version)
}
func (this *IPListDB) ReadItems(offset int64, size int64) (items []*pb.IPItem, err error) {
@@ -210,27 +241,63 @@ func (this *IPListDB) ReadMaxVersion() int64 {
return 0
}
var row = this.selectMaxVersionStmt.QueryRow()
if row == nil {
return 0
// from version table
{
var row = this.selectVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err := row.Scan(&version)
if err == nil {
return version
}
}
var version int64
err := row.Scan(&version)
if err != nil {
return 0
// from items table
{
var row = this.selectMaxItemVersionStmt.QueryRow()
if row == nil {
return 0
}
var version int64
err := row.Scan(&version)
if err != nil {
return 0
}
return version
}
return version
}
// UpdateMaxVersion 修改版本号
func (this *IPListDB) UpdateMaxVersion(version int64) error {
if this.isClosed {
return nil
}
_, err := this.updateVersionStmt.Exec(version)
return err
}
func (this *IPListDB) Close() error {
this.isClosed = true
if this.db != nil {
_ = this.deleteExpiredItemsStmt.Close()
_ = this.deleteItemStmt.Close()
_ = this.insertItemStmt.Close()
_ = this.selectItemsStmt.Close()
_ = this.selectMaxVersionStmt.Close()
for _, stmt := range []*dbs.Stmt{
this.deleteExpiredItemsStmt,
this.deleteItemStmt,
this.insertItemStmt,
this.selectItemsStmt,
this.selectMaxItemVersionStmt, // ipItems table
this.selectVersionStmt, // versions table
this.updateVersionStmt,
} {
if stmt != nil {
_ = stmt.Close()
}
}
return this.db.Close()
}
internal/iplibrary/ip_list_db_test.go
+12
View File
@@ -79,3 +79,15 @@ func TestIPListDB_ReadMaxVersion(t *testing.T) {
}
t.Log(db.ReadMaxVersion())
}
func TestIPListDB_UpdateMaxVersion(t *testing.T) {
db, err := iplibrary.NewIPListDB()
if err != nil {
t.Fatal(err)
}
err = db.UpdateMaxVersion(1027)
if err != nil {
t.Fatal(err)
}
t.Log(db.ReadMaxVersion())
}
internal/iplibrary/manager_ip_list.go
+25 -10
View File
@@ -47,17 +47,20 @@ type IPListManager struct {
db *IPListDB
version int64
pageSize int64
lastVersion int64
fetchPageSize int64
listMap map[int64]*IPList
locker sync.Mutex
isFirstTime bool
}
func NewIPListManager() *IPListManager {
return &IPListManager{
pageSize: 1000,
listMap: map[int64]*IPList{},
fetchPageSize: 5_000,
listMap: map[int64]*IPList{},
isFirstTime: true,
}
}
@@ -117,11 +120,11 @@ func (this *IPListManager) init() {
_ = db.DeleteExpiredItems()
// 本地数据库中最大版本号
this.version = db.ReadMaxVersion()
this.lastVersion = db.ReadMaxVersion()
// 从本地数据库中加载
var offset int64 = 0
var size int64 = 1000
var size int64 = 2_000
for {
items, err := db.ReadItems(offset, size)
var l = len(items)
@@ -148,6 +151,11 @@ func (this *IPListManager) loop() error {
return nil
}
// 第一次同步则打印信息
if this.isFirstTime {
remotelogs.Println("IP_LIST_MANAGER", "initializing ip items ...")
}
for {
hasNext, err := this.fetch()
if err != nil {
@@ -159,6 +167,12 @@ func (this *IPListManager) loop() error {
time.Sleep(1 * time.Second)
}
// 第一次同步则打印信息
if this.isFirstTime {
this.isFirstTime = false
remotelogs.Println("IP_LIST_MANAGER", "finished initializing ip items")
}
return nil
}
@@ -168,8 +182,8 @@ func (this *IPListManager) fetch() (hasNext bool, err error) {
return false, err
}
itemsResp, err := rpcClient.IPItemRPC.ListIPItemsAfterVersion(rpcClient.Context(), &pb.ListIPItemsAfterVersionRequest{
Version: this.version,
Size: this.pageSize,
Version: this.lastVersion,
Size: this.fetchPageSize,
})
if err != nil {
if rpc.IsConnError(err) {
@@ -211,6 +225,7 @@ func (this *IPListManager) DeleteExpiredItems() {
}
}
// 处理IP条目
func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
var changedLists = map[*IPList]zero.Zero{}
for _, item := range items {
@@ -280,8 +295,8 @@ func (this *IPListManager) processItems(items []*pb.IPItem, fromRemote bool) {
if fromRemote {
var latestVersion = items[len(items)-1].Version
if latestVersion > this.version {
this.version = latestVersion
if latestVersion > this.lastVersion {
this.lastVersion = latestVersion
}
}
}