用户状态发生变化时，同步服务状态

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2020, LiuXiangChao
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1 @@
+GoEdge API节点源码

build/build.sh

@@ -1,76 +1,76 @@
 #!/usr/bin/env bash
 
 function build() {
-	ROOT=$(dirname $0)
+	ROOT=$(dirname "$0")
 	NAME="edge-api"
 	DIST=$ROOT/"../dist/${NAME}"
 	OS=${1}
 	ARCH=${2}
 	TAG=${3}
-	NODE_ARCHITECTS=("amd64" "386" "arm64" "mips64" "mips64le")
+	NODE_ARCHITECTS=("amd64" "arm64")
 
-	if [ -z $OS ]; then
+	if [ -z "$OS" ]; then
 		echo "usage: build.sh OS ARCH"
 		exit
 	fi
-	if [ -z $ARCH ]; then
+	if [ -z "$ARCH" ]; then
 		echo "usage: build.sh OS ARCH"
 		exit
 	fi
-	if [ -z $TAG ]; then
+	if [ -z "$TAG" ]; then
 		TAG="community"
 	fi
 
-	VERSION=$(lookup-version $ROOT/../internal/const/const.go)
+	VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
 	ZIP="${NAME}-${OS}-${ARCH}-${TAG}-v${VERSION}.zip"
 
 	# build edge-node
-	NodeVersion=$(lookup-version $ROOT"/../../EdgeNode/internal/const/const.go")
+	NodeVersion=$(lookup-version "$ROOT""/../../EdgeNode/internal/const/const.go")
 	echo "building edge-node v${NodeVersion} ..."
 	EDGE_NODE_BUILD_SCRIPT=$ROOT"/../../EdgeNode/build/build.sh"
-	if [ ! -f $EDGE_NODE_BUILD_SCRIPT ]; then
+	if [ ! -f "$EDGE_NODE_BUILD_SCRIPT" ]; then
 		echo "unable to find edge-node build script 'EdgeNode/build/build.sh'"
 		exit
 	fi
-	cd $ROOT"/../../EdgeNode/build"
+	cd "$ROOT""/../../EdgeNode/build" || exit
 	echo "=============================="
 	for arch in "${NODE_ARCHITECTS[@]}"; do
-		if [ ! -f $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" ]; then
-			./build.sh linux $arch $TAG
+		if [ ! -f "$ROOT""/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" ]; then
+			./build.sh linux "$arch" $TAG
 		else
 			echo "use built node linux/$arch/v${NodeVersion}"
 		fi
 	done
 	echo "=============================="
-	cd -
+	cd - || exit
 
-	rm -f $ROOT/deploy/*.zip
+	rm -f "$ROOT"/deploy/*.zip
 	for arch in "${NODE_ARCHITECTS[@]}"; do
-		cp $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" $ROOT/deploy/edge-node-linux-${arch}-v${NodeVersion}.zip
+		cp "$ROOT""/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" "$ROOT"/deploy/edge-node-linux-"${arch}"-v"${NodeVersion}".zip
 	done
 
 	# build edge-dns
 	if [ "$TAG" = "plus" ]; then
 		DNS_ROOT=$ROOT"/../../EdgeDNS"
-		if [ -d $DNS_ROOT  ]; then
-			DNSNodeVersion=$(lookup-version $ROOT"/../../EdgeDNS/internal/const/const.go")
+		if [ -d "$DNS_ROOT"  ]; then
+			DNSNodeVersion=$(lookup-version "$ROOT""/../../EdgeDNS/internal/const/const.go")
 			echo "building edge-dns ${DNSNodeVersion} ..."
 			EDGE_DNS_NODE_BUILD_SCRIPT=$ROOT"/../../EdgeDNS/build/build.sh"
-			if [ ! -f $EDGE_DNS_NODE_BUILD_SCRIPT ]; then
+			if [ ! -f "$EDGE_DNS_NODE_BUILD_SCRIPT" ]; then
 				echo "unable to find edge-dns build script 'EdgeDNS/build/build.sh'"
 				exit
 			fi
-			cd $ROOT"/../../EdgeDNS/build"
+			cd "$ROOT""/../../EdgeDNS/build" || exit
 			echo "=============================="
-			architects=("amd64")
+			architects=("amd64" "arm64")
 			for arch in "${architects[@]}"; do
-				./build.sh linux $arch $TAG
+				./build.sh linux "$arch" $TAG
 			done
 			echo "=============================="
-			cd -
+			cd - || exit
 
 			for arch in "${architects[@]}"; do
-				cp $ROOT"/../../EdgeDNS/dist/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip" $ROOT/deploy/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip
+				cp "$ROOT""/../../EdgeDNS/dist/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip" "$ROOT"/deploy/edge-dns-linux-"${arch}"-v"${DNSNodeVersion}".zip
 			done
 		fi
 	fi
@@ -78,48 +78,48 @@ function build() {
 	# build sql
 	if [ $TAG = "plus" ]; then
 		echo "building sql ..."
-		${ROOT}/sql.sh
+		"${ROOT}"/sql.sh
 	fi
 
 	# copy files
 	echo "copying ..."
-	if [ ! -d $DIST ]; then
-		mkdir $DIST
-		mkdir $DIST/bin
-		mkdir $DIST/configs
-		mkdir $DIST/logs
+	if [ ! -d "$DIST" ]; then
+		mkdir "$DIST"
+		mkdir "$DIST"/bin
+		mkdir "$DIST"/configs
+		mkdir "$DIST"/logs
 	fi
-	cp $ROOT/configs/api.template.yaml $DIST/configs/
-	cp $ROOT/configs/db.template.yaml $DIST/configs/
-	cp -R $ROOT/deploy $DIST/
-	rm -f $dist/deploy/.gitignore
-	cp -R $ROOT/installers $DIST/
-	cp -R $ROOT/resources $DIST/
-	rm -f $DIST/resources/ipdata/ip2region/global_region.csv
-	rm -f $DIST/resources/ipdata/ip2region/ip.merge.txt
+	cp "$ROOT"/configs/api.template.yaml "$DIST"/configs/
+	cp "$ROOT"/configs/db.template.yaml "$DIST"/configs/
+	cp -R "$ROOT"/deploy "$DIST/"
+	rm -f "$DIST"/deploy/.gitignore
+	cp -R "$ROOT"/installers "$DIST"/
+	cp -R "$ROOT"/resources "$DIST"/
+	rm -f "$DIST"/resources/ipdata/ip2region/global_region.csv
+	rm -f "$DIST"/resources/ipdata/ip2region/ip.merge.txt
 
 	# building edge installer
 	echo "building node installer ..."
-	architects=("amd64" "386" "arm64")
+	architects=("amd64" "arm64")
 	for arch in "${architects[@]}"; do
 		# TODO support arm, mips ...
-		env GOOS=linux GOARCH=${arch} go build -tags $TAG --ldflags="-s -w" -o $ROOT/installers/edge-installer-helper-linux-${arch} $ROOT/../cmd/installer-helper/main.go
+		env GOOS=linux GOARCH="${arch}" go build -trimpath -tags $TAG --ldflags="-s -w" -o "$ROOT"/installers/edge-installer-helper-linux-"${arch}" "$ROOT"/../cmd/installer-helper/main.go
 	done
 
 	# building edge dns installer
 	echo "building dns node installer ..."
-	architects=("amd64" "386" "arm64")
+	architects=("amd64" "arm64")
 	for arch in "${architects[@]}"; do
 		# TODO support arm, mips ...
-		env GOOS=linux GOARCH=${arch} go build -tags $TAG --ldflags="-s -w" -o $ROOT/installers/edge-installer-dns-helper-linux-${arch} $ROOT/../cmd/installer-dns-helper/main.go
+		env GOOS=linux GOARCH="${arch}" go build -trimpath -tags $TAG --ldflags="-s -w" -o "$ROOT"/installers/edge-installer-dns-helper-linux-"${arch}" "$ROOT"/../cmd/installer-dns-helper/main.go
 	done
 
 	# building api node
-	env GOOS=$OS GOARCH=$ARCH go build -tags $TAG --ldflags="-s -w" -o $DIST/bin/edge-api $ROOT/../cmd/edge-api/main.go
+	env GOOS="$OS" GOARCH="$ARCH" go build -trimpath -tags $TAG --ldflags="-s -w" -o "$DIST"/bin/edge-api "$ROOT"/../cmd/edge-api/main.go
 
 	# delete hidden files
-	find $DIST -name ".DS_Store" -delete
-	find $DIST -name ".gitignore" -delete
+	find "$DIST" -name ".DS_Store" -delete
+	find "$DIST" -name ".gitignore" -delete
 
 	echo "zip files"
 	cd "${DIST}/../" || exit
@@ -135,15 +135,15 @@ function build() {
 
 function lookup-version() {
 	FILE=$1
-	VERSION_DATA=$(cat $FILE)
+	VERSION_DATA=$(cat "$FILE")
 	re="Version[ ]+=[ ]+\"([0-9.]+)\""
 	if [[ $VERSION_DATA =~ $re ]]; then
 		VERSION=${BASH_REMATCH[1]}
-		echo $VERSION
+		echo "$VERSION"
 	else
 		echo "could not match version"
 		exit
 	fi
 }
 
-build $1 $2 $3
+build "$1" "$2" "$3"

cmd/edge-api/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"github.com/TeaOSLab/EdgeAPI/internal/apps"
 	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
@@ -11,7 +12,9 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
 	"github.com/iwind/gosock/pkg/gosock"
+	"io/ioutil"
 	"log"
 	"os"
 )
@@ -20,12 +23,12 @@ func main() {
 	if !Tea.IsTesting() {
 		Tea.Env = "prod"
 	}
-	app := apps.NewAppCmd()
+	var app = apps.NewAppCmd()
 	app.Version(teaconst.Version)
 	app.Product(teaconst.ProductName)
-	app.Usage(teaconst.ProcessName + " [start|stop|restart|setup|upgrade|service|daemon]")
+	app.Usage(teaconst.ProcessName + " [start|stop|restart|setup|upgrade|service|daemon|issues]")
 	app.On("setup", func() {
-		setupCmd := setup.NewSetupFromCmd()
+		var setupCmd = setup.NewSetupFromCmd()
 		err := setupCmd.Run()
 		result := maps.Map{}
 		if err != nil {
@@ -97,6 +100,82 @@ func main() {
 			}
 		}
 	})
+	app.On("db.stmt.prepare", func() {
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{Code: "db.stmt.prepare"})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+		} else {
+			var isOn = maps.NewMap(reply.Params).GetBool("isOn")
+			if isOn {
+				fmt.Println("show statements: on")
+			} else {
+				fmt.Println("show statements: off")
+			}
+		}
+	})
+	app.On("db.stmt.count", func() {
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{Code: "db.stmt.count"})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+		} else {
+			var count = maps.NewMap(reply.Params).GetInt("count")
+			fmt.Println("prepared statements count: " + types.String(count))
+		}
+	})
+	app.On("issues", func() {
+		var flagSet = flag.NewFlagSet("issues", flag.ExitOnError)
+		var formatJSON = false
+		flagSet.BoolVar(&formatJSON, "json", false, "")
+		_ = flagSet.Parse(os.Args[2:])
+
+		data, err := ioutil.ReadFile(Tea.LogFile("issues.log"))
+		if err != nil {
+			if formatJSON {
+				fmt.Print("[]")
+			} else {
+				fmt.Println("no issues yet")
+			}
+		} else {
+			var issueMaps = []maps.Map{}
+			err = json.Unmarshal(data, &issueMaps)
+			if err != nil {
+				if formatJSON {
+					fmt.Print("[]")
+				} else {
+					fmt.Println("no issues yet")
+				}
+			} else {
+				if formatJSON {
+					fmt.Print(string(data))
+				} else {
+					if len(issueMaps) == 0 {
+						fmt.Println("no issues yet")
+					} else {
+						for i, issue := range issueMaps {
+							fmt.Println("issue " + types.String(i+1) + ": " + issue.GetString("message"))
+						}
+					}
+				}
+			}
+		}
+	})
+	app.On("instance", func() {
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{Code: "instance"})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+		} else {
+			replyJSON, err := json.MarshalIndent(reply.Params, "", "  ")
+			if err != nil {
+				fmt.Println("[ERROR]marshal result failed: " + err.Error())
+			} else {
+				fmt.Println(string(replyJSON))
+			}
+		}
+	})
+
 	app.Run(func() {
 		nodes.NewAPINode().Start()
 	})

dist/.gitignore

@@ -1 +1,2 @@
-*.zip
+*.zip
+edge-api

go.mod

@@ -1,6 +1,6 @@
 module github.com/TeaOSLab/EdgeAPI
 
-go 1.16
+go 1.18
 
 replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
 
@@ -12,9 +12,8 @@ require (
 	github.com/go-acme/lego/v4 v4.5.2
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang/protobuf v1.5.2
-	github.com/iwind/TeaGo v0.0.0-20220321132348-7da816422f25 // indirect
-	github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/iwind/TeaGo v0.0.0-20220408111647-f36b9bba3570
+	github.com/iwind/gosock v0.0.0-20220505115348-f88412125a62
 	github.com/mozillazg/go-pinyin v0.18.0
 	github.com/pkg/sftp v1.12.0
 	github.com/shirou/gopsutil/v3 v3.22.2
@@ -24,3 +23,25 @@ require (
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
+
+require (
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/tklauser/go-sysconf v0.3.9 // indirect
+	github.com/tklauser/numcpus v0.3.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+)

go.sum

@@ -66,7 +66,6 @@ github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7F
 github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -238,14 +237,12 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/infobloxopen/infoblox-go-client v1.1.1/go.mod h1:BXiw7S2b9qJoM8MS40vfgCNB2NLHGusk1DtO16BD9zI=
 github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
 github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
-github.com/iwind/TeaGo v0.0.0-20220321112016-5a2cd71d3151 h1:jksmjwlGC8QMpyHZmzxr7J+3NeMOr9Zy2+yNJxVSIjI=
-github.com/iwind/TeaGo v0.0.0-20220321112016-5a2cd71d3151/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
-github.com/iwind/TeaGo v0.0.0-20220321131553-fd7b112ba7e7 h1:gdMQZQk/aXfNuKuWCBQhP3byy5Dr8XHMe5+GXdGHcPQ=
-github.com/iwind/TeaGo v0.0.0-20220321131553-fd7b112ba7e7/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
-github.com/iwind/TeaGo v0.0.0-20220321132348-7da816422f25 h1:UpJ52iq8FEz2OeaXFhW1kuYeqVRUQ/5N+NVHvVuTnvw=
-github.com/iwind/TeaGo v0.0.0-20220321132348-7da816422f25/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
+github.com/iwind/TeaGo v0.0.0-20220408111647-f36b9bba3570 h1:zqz2FiMMkSHXWO1EsTRJDPTwX9xQ4uuyD5GAE4JGlhM=
+github.com/iwind/TeaGo v0.0.0-20220408111647-f36b9bba3570/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
 github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
 github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
+github.com/iwind/gosock v0.0.0-20220505115348-f88412125a62 h1:HJH6RDheAY156DnIfJSD/bEvqyXzsZuE2gzs8PuUjoo=
+github.com/iwind/gosock v0.0.0-20220505115348-f88412125a62/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
 github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jarcoal/httpmock v1.0.6/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -561,7 +558,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

internal/accesslogs/storage_base.go

@@ -12,8 +12,9 @@ import (
 )
 
 type BaseStorage struct {
-	isOk    bool
-	version int
+	isOk         bool
+	version      int
+	firewallOnly bool
 }
 
 func (this *BaseStorage) SetVersion(version int) {
@@ -32,6 +33,10 @@ func (this *BaseStorage) SetOk(isOk bool) {
 	this.isOk = isOk
 }
 
+func (this *BaseStorage) SetFirewallOnly(firewallOnly bool) {
+	this.firewallOnly = firewallOnly
+}
+
 // Marshal 对日志进行编码
 func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) {
 	return json.Marshal(accessLog)
@@ -39,7 +44,7 @@ func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) {
 
 // FormatVariables 格式化字符串中的变量
 func (this *BaseStorage) FormatVariables(s string) string {
-	now := time.Now()
+	var now = time.Now()
 	return configutils.ParseVariables(s, func(varName string) (value string) {
 		switch varName {
 		case "year":

internal/accesslogs/storage_command.go

@@ -61,6 +61,10 @@ func (this *CommandStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 		return err
 	}
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
+
 		data, err := this.Marshal(accessLog)
 		if err != nil {
 			logs.Error(err)

internal/accesslogs/storage_es.go

@@ -59,6 +59,10 @@ func (this *ESStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	indexName := this.FormatVariables(this.config.Index)
 	typeName := this.FormatVariables(this.config.MappingType)
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
+
 		if len(accessLog.RequestId) == 0 {
 			continue
 		}

internal/accesslogs/storage_file.go

@@ -57,6 +57,9 @@ func (this *FileStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	defer this.writeLocker.Unlock()
 
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
 		data, err := this.Marshal(accessLog)
 		if err != nil {
 			logs.Error(err)

internal/accesslogs/storage_interface.go

@@ -12,6 +12,9 @@ type StorageInterface interface {
 	// SetVersion 设置版本
 	SetVersion(version int)
 
+	// SetFirewallOnly 设置是否只处理防火墙相关的访问日志
+	SetFirewallOnly(firewallOnly bool)
+
 	IsOk() bool
 
 	SetOk(ok bool)

internal/accesslogs/storage_manager.go

@@ -101,6 +101,7 @@ func (this *StorageManager) Loop() error {
 				}
 
 				storage.SetVersion(types.Int(policy.Version))
+				storage.SetFirewallOnly(policy.FirewallOnly == 1)
 				err := storage.Start()
 				if err != nil {
 					remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error())
@@ -116,6 +117,7 @@ func (this *StorageManager) Loop() error {
 				continue
 			}
 			storage.SetVersion(types.Int(policy.Version))
+			storage.SetFirewallOnly(policy.FirewallOnly == 1)
 			this.storageMap[policyId] = storage
 			err = storage.Start()
 			if err != nil {

internal/accesslogs/storage_syslog.go

@@ -1,7 +1,9 @@
 package accesslogs
 
 import (
+	"bytes"
 	"errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/iwind/TeaGo/logs"
@@ -95,7 +97,10 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 
 	args = append(args, "-S", "10240")
 
-	cmd := exec.Command(this.exe, args...)
+	var cmd = exec.Command(this.exe, args...)
+	var stderrBuffer = &bytes.Buffer{}
+	cmd.Stderr = stderrBuffer
+
 	w, err := cmd.StdinPipe()
 	if err != nil {
 		return err
@@ -106,9 +111,12 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	}
 
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
 		data, err := this.Marshal(accessLog)
 		if err != nil {
-			logs.Error(err)
+			remotelogs.Error("ACCESS_LOG_POLICY_SYSLOG", "marshal accesslog failed: "+err.Error())
 			continue
 		}
 		_, err = w.Write(data)
@@ -118,14 +126,15 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 
 		_, err = w.Write([]byte("\n"))
 		if err != nil {
-			logs.Error(err)
+			remotelogs.Error("ACCESS_LOG_POLICY_SYSLOG", "write accesslog failed: "+err.Error())
 		}
 	}
 
 	_ = w.Close()
+
 	err = cmd.Wait()
 	if err != nil {
-		return err
+		return errors.New("send syslog failed: " + err.Error() + ", stderr: " + stderrBuffer.String())
 	}
 
 	return nil

internal/accesslogs/storage_tcp.go

@@ -60,6 +60,9 @@ func (this *TCPStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	defer this.writeLocker.Unlock()
 
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
 		data, err := this.Marshal(accessLog)
 		if err != nil {
 			logs.Error(err)

internal/apps/log_writer.go

@@ -1,51 +1,108 @@
 package apps
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/sizes"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/files"
-	"github.com/iwind/TeaGo/logs"
-	"github.com/iwind/TeaGo/utils/time"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
 )
 
 type LogWriter struct {
-	fileAppender *files.Appender
+	fp *os.File
+	c  chan string
 }
 
 func (this *LogWriter) Init() {
 	// 创建目录
-	dir := files.NewFile(Tea.LogDir())
+	var dir = files.NewFile(Tea.LogDir())
 	if !dir.Exists() {
 		err := dir.Mkdir()
 		if err != nil {
-			log.Println("[error]" + err.Error())
+			log.Println("[LOG]create log dir failed: " + err.Error())
 		}
 	}
 
-	logFile := files.NewFile(Tea.LogFile("run.log"))
-
 	// 打开要写入的日志文件
-	appender, err := logFile.Appender()
+	var logPath = Tea.LogFile("run.log")
+	fp, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 	if err != nil {
-		logs.Error(err)
+		log.Println("[LOG]open log file failed: " + err.Error())
 	} else {
-		this.fileAppender = appender
+		this.fp = fp
+	}
+
+	this.c = make(chan string, 1024)
+
+	// 异步写入文件
+	var maxFileSize = 2 * sizes.G // 文件最大尺寸，超出此尺寸则清空
+	if fp != nil {
+		goman.New(func() {
+			var totalSize int64 = 0
+			stat, err := fp.Stat()
+			if err == nil {
+				totalSize = stat.Size()
+			}
+
+			for message := range this.c {
+				totalSize += int64(len(message))
+				_, err := fp.WriteString(timeutil.Format("Y/m/d H:i:s ") + message + "\n")
+				if err != nil {
+					log.Println("[LOG]write log failed: " + err.Error())
+				} else {
+					// 如果太大则Truncate
+					if totalSize > maxFileSize {
+						_ = fp.Truncate(0)
+						totalSize = 0
+					}
+				}
+			}
+		})
 	}
 }
 
 func (this *LogWriter) Write(message string) {
-	log.Println(message)
+	backgroundEnv, _ := os.LookupEnv("EdgeBackground")
+	if backgroundEnv != "on" {
+		// 文件和行号
+		var file string
+		var line int
+		if Tea.IsTesting() {
+			var callDepth = 3
+			var ok bool
+			_, file, line, ok = runtime.Caller(callDepth)
+			if ok {
+				file = this.packagePath(file)
+			}
+		}
 
-	if this.fileAppender != nil {
-		_, err := this.fileAppender.AppendString(timeutil.Format("Y/m/d H:i:s ") + message + "\n")
-		if err != nil {
-			log.Println("[error]" + err.Error())
+		if len(file) > 0 {
+			log.Println(message + " (" + file + ":" + strconv.Itoa(line) + ")")
+		} else {
+			log.Println(message)
 		}
 	}
+
+	this.c <- message
 }
 
 func (this *LogWriter) Close() {
-	if this.fileAppender != nil {
-		_ = this.fileAppender.Close()
+	if this.fp != nil {
+		_ = this.fp.Close()
 	}
+
+	close(this.c)
+}
+
+func (this *LogWriter) packagePath(path string) string {
+	var pieces = strings.Split(path, "/")
+	if len(pieces) >= 2 {
+		return strings.Join(pieces[len(pieces)-2:], "/")
+	}
+	return path
 }

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "0.4.6"
+	Version = "0.4.10"
 
 	ProductName   = "Edge API"
 	ProcessName   = "edge-api"
@@ -18,13 +18,13 @@ const (
 
 	// 其他节点版本号，用来检测是否有需要升级的节点
 
-	NodeVersion          = "0.4.5"
-	UserNodeVersion      = "0.3.2"
+	NodeVersion          = "0.4.10"
+	UserNodeVersion      = "0.3.6"
 	AuthorityNodeVersion = "0.0.2"
-	MonitorNodeVersion   = "0.0.3"
-	DNSNodeVersion       = "0.2.1"
-	ReportNodeVersion    = "0.1.0"
+	MonitorNodeVersion   = "0.0.4"
+	DNSNodeVersion       = "0.2.4"
+	ReportNodeVersion    = "0.1.1"
 
 	// SQLVersion SQL版本号
-	SQLVersion = "5"
+	SQLVersion = "2"
 )

internal/const/vars.go

@@ -2,9 +2,18 @@
 
 package teaconst
 
-var (
-	IsPlus         = false
-	MaxNodes int32 = 0
-	NodeId   int64 = 0
-	Debug          = false
+import (
+	"crypto/sha1"
+	"fmt"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	"time"
+)
+
+var (
+	IsPlus             = false
+	MaxNodes     int32 = 0
+	NodeId       int64 = 0
+	Debug              = false
+	InstanceCode       = fmt.Sprintf("%x", sha1.Sum([]byte("INSTANCE"+types.String(time.Now().UnixNano())+"@"+types.String(rands.Int64()))))
 )

internal/db/db_test.go

@@ -3,6 +3,7 @@ package db
 import (
 	"database/sql"
 	"database/sql/driver"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
@@ -42,3 +43,13 @@ func TestDB_Instance(t *testing.T) {
 	}
 	time.Sleep(100 * time.Second)
 }
+
+func TestDB_Reuse(t *testing.T) {
+	var dao = models.NewVersionDAO()
+	for i := 0; i < 20_000; i++ {
+		_, _, err := dao.Query(nil).Attr("version", i).Reuse(true).FindOne()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}

internal/db/models/accounts/user_account_log_dao.go

@@ -2,6 +2,7 @@ package accounts
 
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/userconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -79,7 +80,7 @@ func (this *UserAccountLogDAO) CountAccountLogs(tx *dbs.Tx, userId int64, accoun
 	}
 	if len(keyword) > 0 {
 		query.Where("(userId IN (SELECT id FROM " + models.SharedUserDAO.Table + " WHERE state=1 AND (username LIKE :keyword OR fullname LIKE :keyword)) OR description LIKE :keyword)")
-		query.Param("keyword", "%"+keyword+"%")
+		query.Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(eventType) > 0 {
 		query.Attr("eventType", eventType)
@@ -98,7 +99,7 @@ func (this *UserAccountLogDAO) ListAccountLogs(tx *dbs.Tx, userId int64, account
 	}
 	if len(keyword) > 0 {
 		query.Where("(userId IN (SELECT id FROM " + models.SharedUserDAO.Table + " WHERE state=1 AND (username LIKE :keyword OR fullname LIKE :keyword)) OR description LIKE :keyword)")
-		query.Param("keyword", "%"+keyword+"%")
+		query.Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(eventType) > 0 {
 		query.Attr("eventType", eventType)

internal/db/models/acme/acme_authentication_dao.go

@@ -29,7 +29,7 @@ func init() {
 
 // 创建认证信息
 func (this *ACMEAuthenticationDAO) CreateAuth(tx *dbs.Tx, taskId int64, domain string, token string, key string) error {
-	op := NewACMEAuthenticationOperator()
+	var op = NewACMEAuthenticationOperator()
 	op.TaskId = taskId
 	op.Domain = domain
 	op.Token = token

internal/db/models/acme/acme_task_dao.go

@@ -125,11 +125,11 @@ func (this *ACMETaskDAO) CountAllEnabledACMETasks(tx *dbs.Tx, adminId int64, use
 
 	if len(keyword) > 0 {
 		query.Where("(domains LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(keyword) > 0 {
 		query.Where("domains LIKE :keyword").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	return query.State(ACMETaskStateEnabled).
@@ -155,7 +155,7 @@ func (this *ACMETaskDAO) ListEnabledACMETasks(tx *dbs.Tx, adminId int64, userId
 	}
 	if len(keyword) > 0 {
 		query.Where("(domains LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(ACMETaskStateEnabled).
@@ -169,7 +169,7 @@ func (this *ACMETaskDAO) ListEnabledACMETasks(tx *dbs.Tx, adminId int64, userId
 
 // CreateACMETask 创建任务
 func (this *ACMETaskDAO) CreateACMETask(tx *dbs.Tx, adminId int64, userId int64, authType acmeutils.AuthType, acmeUserId int64, dnsProviderId int64, dnsDomain string, domains []string, autoRenew bool, authURL string) (int64, error) {
-	op := NewACMETaskOperator()
+	var op = NewACMETaskOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.AuthType = authType
@@ -204,7 +204,7 @@ func (this *ACMETaskDAO) UpdateACMETask(tx *dbs.Tx, acmeTaskId int64, acmeUserId
 		return errors.New("invalid acmeTaskId")
 	}
 
-	op := NewACMETaskOperator()
+	var op = NewACMETaskOperator()
 	op.Id = acmeTaskId
 	op.AcmeUserId = acmeUserId
 	op.DnsProviderId = dnsProviderId
@@ -240,7 +240,7 @@ func (this *ACMETaskDAO) UpdateACMETaskCert(tx *dbs.Tx, taskId int64, certId int
 		return errors.New("invalid taskId")
 	}
 
-	op := NewACMETaskOperator()
+	var op = NewACMETaskOperator()
 	op.Id = taskId
 	op.CertId = certId
 	err := this.Save(tx, op)

internal/db/models/acme/acme_task_log_dao.go

@@ -29,7 +29,7 @@ func init() {
 
 // 生成日志
 func (this *ACMETaskLogDAO) CreateACMETaskLog(tx *dbs.Tx, taskId int64, isOk bool, errMsg string) error {
-	op := NewACMETaskLogOperator()
+	var op = NewACMETaskLogOperator()
 	op.TaskId = taskId
 	op.Error = errMsg
 	op.IsOk = isOk

internal/db/models/acme/acme_user_dao.go

@@ -83,7 +83,7 @@ func (this *ACMEUserDAO) CreateACMEUser(tx *dbs.Tx, adminId int64, userId int64,
 	}
 	privateKeyText := base64.StdEncoding.EncodeToString(privateKeyData)
 
-	op := NewACMEUserOperator()
+	var op = NewACMEUserOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.ProviderCode = providerCode
@@ -104,7 +104,7 @@ func (this *ACMEUserDAO) UpdateACMEUser(tx *dbs.Tx, acmeUserId int64, descriptio
 	if acmeUserId <= 0 {
 		return errors.New("invalid acmeUserId")
 	}
-	op := NewACMEUserOperator()
+	var op = NewACMEUserOperator()
 	op.Id = acmeUserId
 	op.Description = description
 	err := this.Save(tx, op)
@@ -116,7 +116,7 @@ func (this *ACMEUserDAO) UpdateACMEUserRegistration(tx *dbs.Tx, acmeUserId int64
 	if acmeUserId <= 0 {
 		return errors.New("invalid acmeUserId")
 	}
-	op := NewACMEUserOperator()
+	var op = NewACMEUserOperator()
 	op.Id = acmeUserId
 	op.Registration = registrationJSON
 	err := this.Save(tx, op)

internal/db/models/admin_dao.go

@@ -115,7 +115,7 @@ func (this *AdminDAO) UpdateAdminPassword(tx *dbs.Tx, adminId int64, password st
 	if adminId <= 0 {
 		return errors.New("invalid adminId")
 	}
-	op := NewAdminOperator()
+	var op = NewAdminOperator()
 	op.Id = adminId
 	op.Password = stringutil.Md5(password)
 	err := this.Save(tx, op)
@@ -124,7 +124,7 @@ func (this *AdminDAO) UpdateAdminPassword(tx *dbs.Tx, adminId int64, password st
 
 // CreateAdmin 创建管理员
 func (this *AdminDAO) CreateAdmin(tx *dbs.Tx, username string, canLogin bool, password string, fullname string, isSuper bool, modulesJSON []byte) (int64, error) {
-	op := NewAdminOperator()
+	var op = NewAdminOperator()
 	op.IsOn = true
 	op.State = AdminStateEnabled
 	op.Username = username
@@ -149,7 +149,7 @@ func (this *AdminDAO) UpdateAdminInfo(tx *dbs.Tx, adminId int64, fullname string
 	if adminId <= 0 {
 		return errors.New("invalid adminId")
 	}
-	op := NewAdminOperator()
+	var op = NewAdminOperator()
 	op.Id = adminId
 	op.Fullname = fullname
 	err := this.Save(tx, op)
@@ -161,7 +161,7 @@ func (this *AdminDAO) UpdateAdmin(tx *dbs.Tx, adminId int64, username string, ca
 	if adminId <= 0 {
 		return errors.New("invalid adminId")
 	}
-	op := NewAdminOperator()
+	var op = NewAdminOperator()
 	op.Id = adminId
 	op.Fullname = fullname
 	op.Username = username
@@ -198,7 +198,7 @@ func (this *AdminDAO) UpdateAdminLogin(tx *dbs.Tx, adminId int64, username strin
 	if adminId <= 0 {
 		return errors.New("invalid adminId")
 	}
-	op := NewAdminOperator()
+	var op = NewAdminOperator()
 	op.Id = adminId
 	op.Username = username
 	if len(password) > 0 {
@@ -213,7 +213,7 @@ func (this *AdminDAO) UpdateAdminModules(tx *dbs.Tx, adminId int64, allowModules
 	if adminId <= 0 {
 		return errors.New("invalid adminId")
 	}
-	op := NewAdminOperator()
+	var op = NewAdminOperator()
 	op.Id = adminId
 	op.Modules = allowModulesJSON
 	err := this.Save(tx, op)

internal/db/models/api_access_token_dao.go

@@ -56,7 +56,7 @@ func (this *APIAccessTokenDAO) GenerateAccessToken(tx *dbs.Tx, adminId int64, us
 	token = rands.String(128) // TODO 增强安全性，将来使用 base64_encode(encrypt(salt+random)) 算法来代替
 	expiresAt = time.Now().Unix() + 7200
 
-	op := NewAPIAccessTokenOperator()
+	var op = NewAPIAccessTokenOperator()
 
 	if accessToken != nil {
 		op.Id = accessToken.(*APIAccessToken).Id

internal/db/models/api_node_dao.go

@@ -3,6 +3,8 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/configs"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -49,16 +51,29 @@ func (this *APINodeDAO) EnableAPINode(tx *dbs.Tx, id int64) error {
 		Pk(id).
 		Set("state", APINodeStateEnabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+	return this.NotifyUpdate(tx, id)
 }
 
 // DisableAPINode 禁用条目
-func (this *APINodeDAO) DisableAPINode(tx *dbs.Tx, id int64) error {
+func (this *APINodeDAO) DisableAPINode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", APINodeStateDisabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+
+	err = this.NotifyUpdate(tx, nodeId)
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleAPI, nodeId)
 }
 
 // FindEnabledAPINode 查找启用中的条目
@@ -119,7 +134,7 @@ func (this *APINodeDAO) CreateAPINode(tx *dbs.Tx, name string, description strin
 		return
 	}
 
-	op := NewAPINodeOperator()
+	var op = NewAPINodeOperator()
 	op.IsOn = isOn
 	op.UniqueId = uniqueId
 	op.Secret = secret
@@ -149,16 +164,33 @@ func (this *APINodeDAO) CreateAPINode(tx *dbs.Tx, name string, description strin
 		return
 	}
 
+	err = this.NotifyUpdate(tx, types.Int64(op.Id))
+	if err != nil {
+		remotelogs.Error("API_NODE_DAO", err.Error())
+	}
+
 	return types.Int64(op.Id), nil
 }
 
 // UpdateAPINode 修改API节点
-func (this *APINodeDAO) UpdateAPINode(tx *dbs.Tx, nodeId int64, name string, description string, httpJSON []byte, httpsJSON []byte, restIsOn bool, restHTTPJSON []byte, restHTTPSJSON []byte, accessAddrsJSON []byte, isOn bool) error {
+func (this *APINodeDAO) UpdateAPINode(tx *dbs.Tx, nodeId int64, name string, description string, httpJSON []byte, httpsJSON []byte, restIsOn bool, restHTTPJSON []byte, restHTTPSJSON []byte, accessAddrsJSON []byte, isOn bool, isPrimary bool) error {
 	if nodeId <= 0 {
 		return errors.New("invalid nodeId")
 	}
 
-	op := NewAPINodeOperator()
+	// 取消别的Primary
+	if isPrimary {
+		err := this.Query(tx).
+			Neq("id", nodeId).
+			Attr("isPrimary", true).
+			Set("isPrimary", false).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+	}
+
+	var op = NewAPINodeOperator()
 	op.Id = nodeId
 	op.Name = name
 	op.Description = description
@@ -191,8 +223,13 @@ func (this *APINodeDAO) UpdateAPINode(tx *dbs.Tx, nodeId int64, name string, des
 		op.AccessAddrs = "[]"
 	}
 
+	op.IsPrimary = isPrimary
+
 	err := this.Save(tx, op)
-	return err
+	if err != nil {
+		return err
+	}
+	return this.NotifyUpdate(tx, nodeId)
 }
 
 // FindAllEnabledAPINodes 列出所有可用API节点
@@ -294,23 +331,6 @@ func (this *APINodeDAO) UpdateAPINodeStatus(tx *dbs.Tx, apiNodeId int64, statusJ
 	return err
 }
 
-// 生成唯一ID
-func (this *APINodeDAO) genUniqueId(tx *dbs.Tx) (string, error) {
-	for {
-		uniqueId := rands.HexString(32)
-		ok, err := this.Query(tx).
-			Attr("uniqueId", uniqueId).
-			Exist()
-		if err != nil {
-			return "", err
-		}
-		if ok {
-			continue
-		}
-		return uniqueId, nil
-	}
-}
-
 // CountAllLowerVersionNodes 计算所有节点中低于某个版本的节点数量
 func (this *APINodeDAO) CountAllLowerVersionNodes(tx *dbs.Tx, version string) (int64, error) {
 	return this.Query(tx).
@@ -384,3 +404,114 @@ func (this *APINodeDAO) FindAllEnabledAPIAccessIPs(tx *dbs.Tx, cacheMap *utils.C
 
 	return result, nil
 }
+
+// CheckAPINodeIsPrimary 检查当前节点是否为Primary节点
+func (this *APINodeDAO) CheckAPINodeIsPrimary(tx *dbs.Tx) (bool, error) {
+	config, err := configs.SharedAPIConfig()
+	if err != nil {
+		return false, err
+	}
+
+	isPrimary, err := this.Query(tx).
+		State(APINodeStateEnabled).
+		Attr("uniqueId", config.NodeId).
+		Attr("isPrimary", true).
+		Exist()
+	if err != nil {
+		return false, err
+	}
+
+	if isPrimary {
+		return true, nil
+	}
+
+	// 检查是否有别的Primary节点
+	count, err := this.Query(tx).
+		State(APINodeStateEnabled).
+		Attr("isOn", true).
+		Attr("isPrimary", true).
+		Count()
+	if err != nil {
+		return false, err
+	}
+
+	if count == 0 {
+		err = this.ResetPrimaryAPINode(tx)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+// CheckAPINodeIsPrimaryWithoutErr 检查当前节点是否为Primary节点，并忽略错误
+func (this *APINodeDAO) CheckAPINodeIsPrimaryWithoutErr() bool {
+	b, err := this.CheckAPINodeIsPrimary(nil)
+	return b && err == nil
+}
+
+// ResetPrimaryAPINode 重置Primary节点
+func (this *APINodeDAO) ResetPrimaryAPINode(tx *dbs.Tx) error {
+	// 当前是否有Primary节点
+	apiNode, err := this.Query(tx).
+		State(APINodeStateEnabled).
+		Attr("isOn", true).
+		Attr("isPrimary", true).
+		Find()
+	if err != nil {
+		return err
+	}
+	if apiNode == nil {
+		// 选择一个作为Primary
+		// TODO 将来需要考虑API节点离线的情况
+		apiNodeId, err := this.Query(tx).
+			State(APINodeStateEnabled).
+			Attr("isOn", true).
+			ResultPk().
+			FindInt64Col(0)
+		if err != nil {
+			return err
+		}
+		if apiNodeId > 0 {
+			err = this.Query(tx).
+				Pk(apiNodeId).
+				Set("isPrimary", true).
+				UpdateQuickly()
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// NotifyUpdate 通知变更
+func (this *APINodeDAO) NotifyUpdate(tx *dbs.Tx, apiNodeId int64) error {
+	// suppress IDE warning
+	_ = apiNodeId
+
+	err := this.ResetPrimaryAPINode(tx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// 生成唯一ID
+func (this *APINodeDAO) genUniqueId(tx *dbs.Tx) (string, error) {
+	for {
+		uniqueId := rands.HexString(32)
+		ok, err := this.Query(tx).
+			Attr("uniqueId", uniqueId).
+			Exist()
+		if err != nil {
+			return "", err
+		}
+		if ok {
+			continue
+		}
+		return uniqueId, nil
+	}
+}

internal/db/models/api_node_dao_test.go

@@ -34,6 +34,16 @@ func TestAPINodeDAO_FindAllEnabledAPIAccessIPs(t *testing.T) {
 	t.Log(NewAPINodeDAO().FindAllEnabledAPIAccessIPs(nil, cacheMap))
 }
 
+func TestAPINodeDAO_CheckAPINodeIsPrimary(t *testing.T) {
+	var dao = NewAPINodeDAO()
+	t.Log(dao.CheckAPINodeIsPrimary(nil))
+}
+
+func TestAPINodeDAO_ResetPrimaryAPINode(t *testing.T) {
+	var dao = NewAPINodeDAO()
+	t.Log(dao.ResetPrimaryAPINode(nil))
+}
+
 func BenchmarkAPINodeDAO_New(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 	for i := 0; i < b.N; i++ {

internal/db/models/api_node_model.go

@@ -23,6 +23,7 @@ type APINode struct {
 	AdminId     uint32   `field:"adminId"`     // 管理员ID
 	Weight      uint32   `field:"weight"`      // 权重
 	Status      dbs.JSON `field:"status"`      // 运行状态
+	IsPrimary   bool     `field:"isPrimary"`   // 是否为主API节点
 }
 
 type APINodeOperator struct {
@@ -45,6 +46,7 @@ type APINodeOperator struct {
 	AdminId     interface{} // 管理员ID
 	Weight      interface{} // 权重
 	Status      interface{} // 运行状态
+	IsPrimary   interface{} // 是否为主API节点
 }
 
 func NewAPINodeOperator() *APINodeOperator {

internal/db/models/api_token_dao.go

@@ -112,7 +112,7 @@ func (this *ApiTokenDAO) FindEnabledTokenWithRole(tx *dbs.Tx, role string) (*Api
 
 // CreateAPIToken 保存API Token
 func (this *ApiTokenDAO) CreateAPIToken(tx *dbs.Tx, nodeId string, secret string, role nodeconfigs.NodeRole) error {
-	op := NewApiTokenOperator()
+	var op = NewApiTokenOperator()
 	op.NodeId = nodeId
 	op.Secret = secret
 	op.Role = role

internal/db/models/authority/authority_key_dao.go

@@ -42,7 +42,7 @@ func (this *AuthorityKeyDAO) UpdateKey(tx *dbs.Tx, value string, dayFrom string,
 	if err != nil {
 		return err
 	}
-	op := NewAuthorityKeyOperator()
+	var op = NewAuthorityKeyOperator()
 	if one != nil {
 		op.Id = one.(*AuthorityKey).Id
 	}

internal/db/models/authority/authority_node_dao.go

@@ -1,6 +1,7 @@
 package authority
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
@@ -119,7 +120,7 @@ func (this *AuthorityNodeDAO) CreateAuthorityNode(tx *dbs.Tx, name string, descr
 		return
 	}
 
-	op := NewAuthorityNodeOperator()
+	var op = NewAuthorityNodeOperator()
 	op.IsOn = isOn
 	op.UniqueId = uniqueId
 	op.Secret = secret
@@ -140,7 +141,7 @@ func (this *AuthorityNodeDAO) UpdateAuthorityNode(tx *dbs.Tx, nodeId int64, name
 		return errors.New("invalid nodeId")
 	}
 
-	op := NewAuthorityNodeOperator()
+	var op = NewAuthorityNodeOperator()
 	op.Id = nodeId
 	op.Name = name
 	op.Description = description
@@ -188,13 +189,19 @@ func (this *AuthorityNodeDAO) GenUniqueId(tx *dbs.Tx) (string, error) {
 }
 
 // UpdateNodeStatus 更改节点状态
-func (this *AuthorityNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, statusJSON []byte) error {
-	if statusJSON == nil {
+func (this *AuthorityNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, nodeStatus *nodeconfigs.NodeStatus) error {
+	if nodeStatus == nil {
 		return nil
 	}
-	_, err := this.Query(tx).
+
+	nodeStatusJSON, err := json.Marshal(nodeStatus)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.Query(tx).
 		Pk(nodeId).
-		Set("status", string(statusJSON)).
+		Set("status", nodeStatusJSON).
 		Update()
 	return err
 }

internal/db/models/client_browser_dao.go

@@ -125,7 +125,7 @@ func (this *ClientBrowserDAO) CreateBrowser(tx *dbs.Tx, browserName string) (int
 		return browserId, nil
 	}
 
-	op := NewClientBrowserOperator()
+	var op = NewClientBrowserOperator()
 	op.Name = browserName
 	codes := []string{browserName}
 	codesJSON, err := json.Marshal(codes)

internal/db/models/client_system_dao.go

@@ -125,7 +125,7 @@ func (this *ClientSystemDAO) CreateSystem(tx *dbs.Tx, systemName string) (int64,
 		return systemId, nil
 	}
 
-	op := NewClientSystemOperator()
+	var op = NewClientSystemOperator()
 	op.Name = systemName
 
 	codes := []string{systemName}

internal/db/models/db_node_dao.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"github.com/TeaOSLab/EdgeAPI/internal/encrypt"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -49,12 +50,17 @@ func (this *DBNodeDAO) EnableDBNode(tx *dbs.Tx, id int64) error {
 }
 
 // DisableDBNode 禁用条目
-func (this *DBNodeDAO) DisableDBNode(tx *dbs.Tx, id int64) error {
+func (this *DBNodeDAO) DisableDBNode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", DBNodeStateDisabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleDatabase, nodeId)
 }
 
 // FindEnabledDBNode 查找启用中的条目
@@ -103,7 +109,7 @@ func (this *DBNodeDAO) ListEnabledNodes(tx *dbs.Tx, offset int64, size int64) (r
 
 // CreateDBNode 创建节点
 func (this *DBNodeDAO) CreateDBNode(tx *dbs.Tx, isOn bool, name string, description string, host string, port int32, database string, username string, password string, charset string) (int64, error) {
-	op := NewDBNodeOperator()
+	var op = NewDBNodeOperator()
 	op.State = NodeStateEnabled
 	op.IsOn = isOn
 	op.Name = name
@@ -126,7 +132,7 @@ func (this *DBNodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, isOn bool, name stri
 	if nodeId <= 0 {
 		return errors.New("invalid nodeId")
 	}
-	op := NewDBNodeOperator()
+	var op = NewDBNodeOperator()
 	op.Id = nodeId
 	op.IsOn = isOn
 	op.Name = name

internal/db/models/db_node_initializer.go

@@ -56,6 +56,25 @@ func init() {
 	})
 }
 
+func AllAccessLogDBs() []*dbs.DB {
+	accessLogLocker.Lock()
+	defer accessLogLocker.Unlock()
+
+	var result = []*dbs.DB{}
+	for _, db := range accessLogDBMapping {
+		result = append(result, db)
+	}
+
+	if len(result) == 0 {
+		db, _ := dbs.Default()
+		if db != nil {
+			result = append(result, db)
+		}
+	}
+
+	return result
+}
+
 // 获取获取DAO
 func randomHTTPAccessLogDAO() (dao *HTTPAccessLogDAOWrapper) {
 	accessLogLocker.RLock()
@@ -237,7 +256,7 @@ func (this *DBNodeInitializer) loop() error {
 		}
 
 		if db == nil {
-			config := &dbs.DBConfig{
+			var config = &dbs.DBConfig{
 				Driver: "mysql",
 				Dsn:    dsn,
 				Prefix: "edge",
@@ -251,7 +270,7 @@ func (this *DBNodeInitializer) loop() error {
 			// 检查表是否存在
 			// httpAccessLog
 			{
-				tableDef, err := SharedHTTPAccessLogManager.FindTable(db, timeutil.Format("Ymd"), true)
+				tableDef, err := SharedHTTPAccessLogManager.FindLastTable(db, timeutil.Format("Ymd"), true)
 				if err != nil {
 					remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
 

internal/db/models/dns/dns_domain_dao.go

@@ -91,7 +91,7 @@ func (this *DNSDomainDAO) FindDNSDomainName(tx *dbs.Tx, id int64) (string, error
 
 // CreateDomain 创建域名
 func (this *DNSDomainDAO) CreateDomain(tx *dbs.Tx, adminId int64, userId int64, providerId int64, name string) (int64, error) {
-	op := NewDNSDomainOperator()
+	var op = NewDNSDomainOperator()
 	op.ProviderId = providerId
 	op.AdminId = adminId
 	op.UserId = userId
@@ -111,7 +111,7 @@ func (this *DNSDomainDAO) UpdateDomain(tx *dbs.Tx, domainId int64, name string,
 	if domainId <= 0 {
 		return errors.New("invalid domainId")
 	}
-	op := NewDNSDomainOperator()
+	var op = NewDNSDomainOperator()
 	op.Id = domainId
 	op.Name = name
 	op.IsOn = isOn
@@ -146,7 +146,7 @@ func (this *DNSDomainDAO) UpdateDomainData(tx *dbs.Tx, domainId int64, data stri
 	if domainId <= 0 {
 		return errors.New("invalid domainId")
 	}
-	op := NewDNSDomainOperator()
+	var op = NewDNSDomainOperator()
 	op.Id = domainId
 	op.Data = data
 	err := this.Save(tx, op)
@@ -158,7 +158,7 @@ func (this *DNSDomainDAO) UpdateDomainRecords(tx *dbs.Tx, domainId int64, record
 	if domainId <= 0 {
 		return errors.New("invalid domainId")
 	}
-	op := NewDNSDomainOperator()
+	var op = NewDNSDomainOperator()
 	op.Id = domainId
 	op.Records = recordsJSON
 	op.DataUpdatedAt = time.Now().Unix()
@@ -171,7 +171,7 @@ func (this *DNSDomainDAO) UpdateDomainRoutes(tx *dbs.Tx, domainId int64, routesJ
 	if domainId <= 0 {
 		return errors.New("invalid domainId")
 	}
-	op := NewDNSDomainOperator()
+	var op = NewDNSDomainOperator()
 	op.Id = domainId
 	op.Routes = routesJSON
 	op.DataUpdatedAt = time.Now().Unix()

internal/db/models/dns/dns_provider_dao.go

@@ -68,7 +68,7 @@ func (this *DNSProviderDAO) FindEnabledDNSProvider(tx *dbs.Tx, id int64) (*DNSPr
 
 // CreateDNSProvider 创建服务商
 func (this *DNSProviderDAO) CreateDNSProvider(tx *dbs.Tx, adminId int64, userId int64, providerType string, name string, apiParamsJSON []byte) (int64, error) {
-	op := NewDNSProviderOperator()
+	var op = NewDNSProviderOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.Type = providerType
@@ -90,7 +90,7 @@ func (this *DNSProviderDAO) UpdateDNSProvider(tx *dbs.Tx, dnsProviderId int64, n
 		return errors.New("invalid dnsProviderId")
 	}
 
-	op := NewDNSProviderOperator()
+	var op = NewDNSProviderOperator()
 	op.Id = dnsProviderId
 	op.Name = name
 
@@ -107,22 +107,39 @@ func (this *DNSProviderDAO) UpdateDNSProvider(tx *dbs.Tx, dnsProviderId int64, n
 }
 
 // CountAllEnabledDNSProviders 计算服务商数量
-func (this *DNSProviderDAO) CountAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string) (int64, error) {
+func (this *DNSProviderDAO) CountAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, domain string, providerType string) (int64, error) {
 	var query = dbutils.NewQuery(tx, this, adminId, userId)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
+
+	if len(domain) > 0 {
+		query.Where("id IN (SELECT providerId FROM " + SharedDNSDomainDAO.Table + " WHERE state=1 AND name=:domain)")
+		query.Param("domain", domain)
+	}
+
+	if len(providerType) > 0 {
+		query.Attr("type", providerType)
+	}
+
 	return query.State(DNSProviderStateEnabled).
 		Count()
 }
 
 // ListEnabledDNSProviders 列出单页服务商
-func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, offset int64, size int64) (result []*DNSProvider, err error) {
+func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, domain string, providerType string, offset int64, size int64) (result []*DNSProvider, err error) {
 	var query = dbutils.NewQuery(tx, this, adminId, userId)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
+	}
+	if len(domain) > 0 {
+		query.Where("id IN (SELECT providerId FROM " + SharedDNSDomainDAO.Table + " WHERE state=1 AND name=:domain)")
+		query.Param("domain", domain)
+	}
+	if len(providerType) > 0 {
+		query.Attr("type", providerType)
 	}
 	_, err = query.
 		State(DNSProviderStateEnabled).

internal/db/models/dns/dns_task_dao.go

@@ -12,10 +12,11 @@ import (
 type DNSTaskType = string
 
 const (
-	DNSTaskTypeClusterChange DNSTaskType = "clusterChange"
-	DNSTaskTypeNodeChange    DNSTaskType = "nodeChange"
-	DNSTaskTypeServerChange  DNSTaskType = "serverChange"
-	DNSTaskTypeDomainChange  DNSTaskType = "domainChange"
+	DNSTaskTypeClusterChange       DNSTaskType = "clusterChange"
+	DNSTaskTypeClusterRemoveDomain DNSTaskType = "clusterRemoveDomain" // 从集群中移除域名
+	DNSTaskTypeNodeChange          DNSTaskType = "nodeChange"
+	DNSTaskTypeServerChange        DNSTaskType = "serverChange"
+	DNSTaskTypeDomainChange        DNSTaskType = "domainChange"
 )
 
 type DNSTaskDAO dbs.DAO
@@ -40,20 +41,21 @@ func init() {
 }
 
 // CreateDNSTask 生成任务
-func (this *DNSTaskDAO) CreateDNSTask(tx *dbs.Tx, clusterId int64, serverId int64, nodeId int64, domainId int64, taskType string) error {
+func (this *DNSTaskDAO) CreateDNSTask(tx *dbs.Tx, clusterId int64, serverId int64, nodeId int64, domainId int64, recordName string, taskType string) error {
 	if clusterId <= 0 && serverId <= 0 && nodeId <= 0 && domainId <= 0 {
 		return nil
 	}
 	err := this.Query(tx).InsertOrUpdateQuickly(maps.Map{
-		"clusterId": clusterId,
-		"serverId":  serverId,
-		"nodeId":    nodeId,
-		"domainId":  domainId,
-		"updatedAt": time.Now().Unix(),
-		"type":      taskType,
-		"isDone":    false,
-		"isOk":      false,
-		"error":     "",
+		"clusterId":  clusterId,
+		"serverId":   serverId,
+		"nodeId":     nodeId,
+		"domainId":   domainId,
+		"recordName": recordName,
+		"updatedAt":  time.Now().Unix(),
+		"type":       taskType,
+		"isDone":     false,
+		"isOk":       false,
+		"error":      "",
 	}, maps.Map{
 		"updatedAt": time.Now().Unix(),
 		"isDone":    false,
@@ -63,24 +65,29 @@ func (this *DNSTaskDAO) CreateDNSTask(tx *dbs.Tx, clusterId int64, serverId int6
 	return err
 }
 
-// CreateClusterTask 生成集群任务
+// CreateClusterTask 生成集群变更任务
 func (this *DNSTaskDAO) CreateClusterTask(tx *dbs.Tx, clusterId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, clusterId, 0, 0, 0, taskType)
+	return this.CreateDNSTask(tx, clusterId, 0, 0, 0, "", taskType)
+}
+
+// CreateClusterRemoveTask 生成集群删除域名任务
+func (this *DNSTaskDAO) CreateClusterRemoveTask(tx *dbs.Tx, clusterId int64, domainId int64, recordName string) error {
+	return this.CreateDNSTask(tx, clusterId, 0, 0, domainId, recordName, DNSTaskTypeClusterRemoveDomain)
 }
 
 // CreateNodeTask 生成节点任务
 func (this *DNSTaskDAO) CreateNodeTask(tx *dbs.Tx, nodeId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, 0, 0, nodeId, 0, taskType)
+	return this.CreateDNSTask(tx, 0, 0, nodeId, 0, "", taskType)
 }
 
 // CreateServerTask 生成服务任务
-func (this *DNSTaskDAO) CreateServerTask(tx *dbs.Tx, serverId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, 0, serverId, 0, 0, taskType)
+func (this *DNSTaskDAO) CreateServerTask(tx *dbs.Tx, clusterId int64, serverId int64, taskType DNSTaskType) error {
+	return this.CreateDNSTask(tx, clusterId, serverId, 0, 0, "", taskType)
 }
 
 // CreateDomainTask 生成域名更新任务
 func (this *DNSTaskDAO) CreateDomainTask(tx *dbs.Tx, domainId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, 0, 0, 0, domainId, taskType)
+	return this.CreateDNSTask(tx, 0, 0, 0, domainId, "", taskType)
 }
 
 // FindAllDoingTasks 查找所有正在执行的任务
@@ -101,6 +108,7 @@ func (this *DNSTaskDAO) FindAllDoingOrErrorTasks(tx *dbs.Tx, nodeClusterId int64
 	}
 	_, err = query.
 		Where("(isDone=0 OR (isDone=1 AND isOk=0))").
+		Asc("updatedAt").
 		AscPk().
 		Slice(&result).
 		FindAll()
@@ -135,7 +143,7 @@ func (this *DNSTaskDAO) UpdateDNSTaskError(tx *dbs.Tx, taskId int64, err string)
 	if taskId <= 0 {
 		return errors.New("invalid taskId")
 	}
-	op := NewDNSTaskOperator()
+	var op = NewDNSTaskOperator()
 	op.Id = taskId
 	op.IsDone = true
 	op.Error = err
@@ -148,7 +156,7 @@ func (this *DNSTaskDAO) UpdateDNSTaskDone(tx *dbs.Tx, taskId int64) error {
 	if taskId <= 0 {
 		return errors.New("invalid taskId")
 	}
-	op := NewDNSTaskOperator()
+	var op = NewDNSTaskOperator()
 	op.Id = taskId
 	op.IsDone = true
 	op.IsOk = true

internal/db/models/dns/dns_task_model.go

@@ -1,30 +1,32 @@
 package dns
 
-// DNS更新任务
+// DNSTask DNS更新任务
 type DNSTask struct {
-	Id        uint64 `field:"id"`        // ID
-	ClusterId uint32 `field:"clusterId"` // 集群ID
-	ServerId  uint32 `field:"serverId"`  // 服务ID
-	NodeId    uint32 `field:"nodeId"`    // 节点ID
-	DomainId  uint32 `field:"domainId"`  // 域名ID
-	Type      string `field:"type"`      // 任务类型
-	UpdatedAt uint64 `field:"updatedAt"` // 更新时间
-	IsDone    bool   `field:"isDone"`    // 是否已完成
-	IsOk      bool   `field:"isOk"`      // 是否成功
-	Error     string `field:"error"`     // 错误信息
+	Id         uint64 `field:"id"`         // ID
+	ClusterId  uint32 `field:"clusterId"`  // 集群ID
+	ServerId   uint32 `field:"serverId"`   // 服务ID
+	NodeId     uint32 `field:"nodeId"`     // 节点ID
+	DomainId   uint32 `field:"domainId"`   // 域名ID
+	RecordName string `field:"recordName"` // 记录名
+	Type       string `field:"type"`       // 任务类型
+	UpdatedAt  uint64 `field:"updatedAt"`  // 更新时间
+	IsDone     bool   `field:"isDone"`     // 是否已完成
+	IsOk       bool   `field:"isOk"`       // 是否成功
+	Error      string `field:"error"`      // 错误信息
 }
 
 type DNSTaskOperator struct {
-	Id        interface{} // ID
-	ClusterId interface{} // 集群ID
-	ServerId  interface{} // 服务ID
-	NodeId    interface{} // 节点ID
-	DomainId  interface{} // 域名ID
-	Type      interface{} // 任务类型
-	UpdatedAt interface{} // 更新时间
-	IsDone    interface{} // 是否已完成
-	IsOk      interface{} // 是否成功
-	Error     interface{} // 错误信息
+	Id         interface{} // ID
+	ClusterId  interface{} // 集群ID
+	ServerId   interface{} // 服务ID
+	NodeId     interface{} // 节点ID
+	DomainId   interface{} // 域名ID
+	RecordName interface{} // 记录名
+	Type       interface{} // 任务类型
+	UpdatedAt  interface{} // 更新时间
+	IsDone     interface{} // 是否已完成
+	IsOk       interface{} // 是否成功
+	Error      interface{} // 错误信息
 }
 
 func NewDNSTaskOperator() *DNSTaskOperator {

internal/db/models/dns/dnsutils/dns_utils.go

@@ -153,7 +153,7 @@ func CheckClusterDNS(tx *dbs.Tx, cluster *models.NodeCluster) (issues []*pb.DNSI
 		}
 
 		// 检查IP地址
-		ipAddr, _, err := models.SharedNodeIPAddressDAO.FindFirstNodeAccessIPAddress(tx, nodeId, nodeconfigs.NodeRoleNode)
+		ipAddr, _, err := models.SharedNodeIPAddressDAO.FindFirstNodeAccessIPAddress(tx, nodeId, true, nodeconfigs.NodeRoleNode)
 		if err != nil {
 			return nil, err
 		}

internal/db/models/file_chunk_dao.go

@@ -29,9 +29,9 @@ func init() {
 	})
 }
 
-// 创建文件Chunk
+// CreateFileChunk 创建文件Chunk
 func (this *FileChunkDAO) CreateFileChunk(tx *dbs.Tx, fileId int64, data []byte) (int64, error) {
-	op := NewFileChunkOperator()
+	var op = NewFileChunkOperator()
 	op.FileId = fileId
 	op.Data = data
 	err := this.Save(tx, op)
@@ -41,7 +41,7 @@ func (this *FileChunkDAO) CreateFileChunk(tx *dbs.Tx, fileId int64, data []byte)
 	return types.Int64(op.Id), nil
 }
 
-// 列出所有的文件Chunk
+// FindAllFileChunks 列出所有的文件Chunk
 func (this *FileChunkDAO) FindAllFileChunks(tx *dbs.Tx, fileId int64) (result []*FileChunk, err error) {
 	_, err = this.Query(tx).
 		Attr("fileId", fileId).
@@ -51,7 +51,7 @@ func (this *FileChunkDAO) FindAllFileChunks(tx *dbs.Tx, fileId int64) (result []
 	return
 }
 
-// 读取文件的所有片段ID
+// FindAllFileChunkIds 读取文件的所有片段ID
 func (this *FileChunkDAO) FindAllFileChunkIds(tx *dbs.Tx, fileId int64) ([]int64, error) {
 	ones, err := this.Query(tx).
 		Attr("fileId", fileId).
@@ -68,7 +68,7 @@ func (this *FileChunkDAO) FindAllFileChunkIds(tx *dbs.Tx, fileId int64) ([]int64
 	return result, nil
 }
 
-// 删除以前的文件
+// DeleteFileChunks 删除以前的文件
 func (this *FileChunkDAO) DeleteFileChunks(tx *dbs.Tx, fileId int64) error {
 	if fileId <= 0 {
 		return errors.New("invalid fileId")
@@ -79,7 +79,7 @@ func (this *FileChunkDAO) DeleteFileChunks(tx *dbs.Tx, fileId int64) error {
 	return err
 }
 
-// 根据ID查找片段
+// FindFileChunk 根据ID查找片段
 func (this *FileChunkDAO) FindFileChunk(tx *dbs.Tx, chunkId int64) (*FileChunk, error) {
 	one, err := this.Query(tx).
 		Pk(chunkId).

internal/db/models/file_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -33,7 +34,7 @@ func init() {
 	})
 }
 
-// 启用条目
+// EnableFile 启用条目
 func (this *FileDAO) EnableFile(tx *dbs.Tx, id int64) error {
 	_, err := this.Query(tx).
 		Pk(id).
@@ -42,7 +43,7 @@ func (this *FileDAO) EnableFile(tx *dbs.Tx, id int64) error {
 	return err
 }
 
-// 禁用条目
+// DisableFile 禁用条目
 func (this *FileDAO) DisableFile(tx *dbs.Tx, id int64) error {
 	_, err := this.Query(tx).
 		Pk(id).
@@ -51,7 +52,7 @@ func (this *FileDAO) DisableFile(tx *dbs.Tx, id int64) error {
 	return err
 }
 
-// 查找启用中的条目
+// FindEnabledFile 查找启用中的条目
 func (this *FileDAO) FindEnabledFile(tx *dbs.Tx, id int64) (*File, error) {
 	result, err := this.Query(tx).
 		Pk(id).
@@ -63,9 +64,9 @@ func (this *FileDAO) FindEnabledFile(tx *dbs.Tx, id int64) (*File, error) {
 	return result.(*File), err
 }
 
-// 创建文件
-func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, businessType, description string, filename string, size int64, isPublic bool) (int64, error) {
-	op := NewFileOperator()
+// CreateFile 创建文件
+func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, businessType string, description string, filename string, size int64, mimeType string, isPublic bool) (int64, error) {
+	var op = NewFileOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.Type = businessType
@@ -74,6 +75,8 @@ func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, busines
 	op.Size = size
 	op.Filename = filename
 	op.IsPublic = isPublic
+	op.Code = utils.Sha1RandomString()
+	op.MimeType = mimeType
 	err := this.Save(tx, op)
 	if err != nil {
 		return 0, err
@@ -82,7 +85,22 @@ func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, busines
 	return types.Int64(op.Id), nil
 }
 
-// 将文件置为已完成
+// CheckUserFile 检查用户ID
+func (this *FileDAO) CheckUserFile(tx *dbs.Tx, userId int64, fileId int64) error {
+	b, err := this.Query(tx).
+		Pk(fileId).
+		Attr("userId", userId).
+		Exist()
+	if err != nil {
+		return err
+	}
+	if !b {
+		return ErrNotFound
+	}
+	return nil
+}
+
+// UpdateFileIsFinished 将文件置为已完成
 func (this *FileDAO) UpdateFileIsFinished(tx *dbs.Tx, fileId int64) error {
 	_, err := this.Query(tx).
 		Pk(fileId).

internal/db/models/file_model.go

@@ -1,13 +1,15 @@
 package models
 
-// 文件管理
+// File 文件管理
 type File struct {
 	Id          uint32 `field:"id"`          // ID
 	AdminId     uint32 `field:"adminId"`     // 管理员ID
+	Code        string `field:"code"`        // 代号
 	UserId      uint32 `field:"userId"`      // 用户ID
 	Description string `field:"description"` // 文件描述
 	Filename    string `field:"filename"`    // 文件名
 	Size        uint32 `field:"size"`        // 文件尺寸
+	MimeType    string `field:"mimeType"`    // Mime类型
 	CreatedAt   uint64 `field:"createdAt"`   // 创建时间
 	Order       uint32 `field:"order"`       // 排序
 	Type        string `field:"type"`        // 类型
@@ -19,10 +21,12 @@ type File struct {
 type FileOperator struct {
 	Id          interface{} // ID
 	AdminId     interface{} // 管理员ID
+	Code        interface{} // 代号
 	UserId      interface{} // 用户ID
 	Description interface{} // 文件描述
 	Filename    interface{} // 文件名
 	Size        interface{} // 文件尺寸
+	MimeType    interface{} // Mime类型
 	CreatedAt   interface{} // 创建时间
 	Order       interface{} // 排序
 	Type        interface{} // 类型

internal/db/models/http_access_log_dao.go

@@ -3,6 +3,7 @@ package models
 import (
 	"bytes"
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,6 +23,7 @@ import (
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"net"
 	"net/http"
+	"net/url"
 	"regexp"
 	"sort"
 	"strings"
@@ -181,7 +183,7 @@ Loop:
 // CreateHTTPAccessLog 写入单条访问日志
 func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLogDAO, accessLog *pb.HTTPAccessLog) error {
 	var day = timeutil.FormatTime("Ymd", accessLog.Timestamp)
-	tableDef, err := SharedHTTPAccessLogManager.FindTable(dao.Instance, day, true)
+	tableDef, err := SharedHTTPAccessLogManager.FindLastTable(dao.Instance, day, true)
 	if err != nil {
 		return err
 	}
@@ -248,7 +250,9 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLog
 }
 
 // ListAccessLogs 读取往前的 单页访问日志
-func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
+func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx,
+	partition int32,
+	lastRequestId string,
 	size int64,
 	day string,
 	hourFrom string,
@@ -275,18 +279,19 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
 		size = 1000
 	}
 
-	result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
+	result, nextLastRequestId, err = this.listAccessLogs(tx, partition, lastRequestId, size, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	if err != nil || int64(len(result)) < size {
 		return
 	}
 
-	moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
+	moreResult, _, _ := this.listAccessLogs(tx, partition, nextLastRequestId, 1, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	hasMore = len(moreResult) > 0
 	return
 }
 
 // 读取往前的单页访问日志
 func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
+	partition int32,
 	lastRequestId string,
 	size int64,
 	day string,
@@ -309,7 +314,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 		return nil, lastRequestId, nil
 	}
 
-	serverIds := []int64{}
+	var serverIds = []int64{}
 	if userId > 0 {
 		serverIds, err = SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
 		if err != nil {
@@ -321,7 +326,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 	}
 
 	accessLogLocker.RLock()
-	daoList := []*HTTPAccessLogDAOWrapper{}
+	var daoList = []*HTTPAccessLogDAOWrapper{}
 	for _, daoWrapper := range httpAccessLogDAOMapping {
 		daoList = append(daoList, daoWrapper)
 	}
@@ -339,7 +344,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 	if clusterId > 0 {
 		nodeIds, err = SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
 		if err != nil {
-			remotelogs.Error("DBNODE", err.Error())
+			remotelogs.Error("DB_NODE", err.Error())
 			return
 		}
 		sort.Slice(nodeIds, func(i, j int) bool {
@@ -349,32 +354,61 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 
 	// 准备查询
 	var tableQueries = []*accessLogTableQuery{}
+	var maxTableName = ""
 	for _, daoWrapper := range daoList {
 		var instance = daoWrapper.DAO.Instance
-		tableDefs, err := SharedHTTPAccessLogManager.FindTables(instance, day)
+		def, err := SharedHTTPAccessLogManager.FindPartitionTable(instance, day, partition)
 		if err != nil {
 			return nil, "", err
 		}
-		for _, def := range tableDefs {
-			tableQueries = append(tableQueries, &accessLogTableQuery{
-				daoWrapper:         daoWrapper,
-				name:               def.Name,
-				hasRemoteAddrField: def.HasRemoteAddr,
-				hasDomainField:     def.HasDomain,
-			})
+		if !def.Exists {
+			continue
 		}
+
+		if len(maxTableName) == 0 || def.Name > maxTableName {
+			maxTableName = def.Name
+		}
+
+		tableQueries = append(tableQueries, &accessLogTableQuery{
+			daoWrapper:         daoWrapper,
+			name:               def.Name,
+			hasRemoteAddrField: def.HasRemoteAddr,
+			hasDomainField:     def.HasDomain,
+		})
+	}
+
+	// 检查各个分表是否一致
+	if partition < 0 {
+		var newTableQueries = []*accessLogTableQuery{}
+		for _, tableQuery := range tableQueries {
+			if tableQuery.name != maxTableName {
+				continue
+			}
+			newTableQueries = append(newTableQueries, tableQuery)
+		}
+		tableQueries = newTableQueries
+	}
+
+	if len(tableQueries) == 0 {
+		return nil, "", nil
 	}
 
 	var locker = sync.Mutex{}
 
+	// 这里正则表达式中的括号不能轻易变更，因为后面有引用
+	// TODO 支持多个查询条件的组合，比如 status:200 proto:HTTP/1.1
 	var statusPrefixReg = regexp.MustCompile(`status:\s*(\d{3})\b`)
 	var statusRangeReg = regexp.MustCompile(`status:\s*(\d{3})-(\d{3})\b`)
+	var urlReg = regexp.MustCompile(`^(http|https)://`)
+	var requestPathReg = regexp.MustCompile(`requestPath:(\S+)`)
+	var protoReg = regexp.MustCompile(`proto:(\S+)`)
+	var schemeReg = regexp.MustCompile(`scheme:(\S+)`)
 
 	var count = len(tableQueries)
 	var wg = &sync.WaitGroup{}
 	wg.Add(count)
 	for _, tableQuery := range tableQueries {
-		go func(tableQuery *accessLogTableQuery) {
+		go func(tableQuery *accessLogTableQuery, keyword string) {
 			defer wg.Done()
 
 			var dao = tableQuery.daoWrapper.DAO
@@ -462,27 +496,53 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 			}
 
 			if len(keyword) > 0 {
-				// remoteAddr
-				if tableQuery.hasRemoteAddrField && net.ParseIP(keyword) != nil {
+				var isSpecialKeyword = false
+
+				if tableQuery.hasRemoteAddrField && net.ParseIP(keyword) != nil { // ip
+					isSpecialKeyword = true
 					query.Attr("remoteAddr", keyword)
-				} else if tableQuery.hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
+				} else if tableQuery.hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) { // ip:x.x.x.x
+					isSpecialKeyword = true
 					keyword = keyword[3:]
 					pieces := strings.SplitN(keyword, ",", 2)
-					if len(pieces) == 1 || len(pieces[1]) == 0 {
+					if len(pieces) == 1 || len(pieces[1]) == 0 || pieces[0] == pieces[1] {
 						query.Attr("remoteAddr", pieces[0])
 					} else {
 						query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
 					}
-				} else if statusRangeReg.MatchString(keyword) {
+				} else if statusRangeReg.MatchString(keyword) { // status:200-400
+					isSpecialKeyword = true
 					var matches = statusRangeReg.FindStringSubmatch(keyword)
 					query.Between("status", types.Int(matches[1]), types.Int(matches[2]))
-
-					// TODO 处理剩余的关键词
-				} else if statusPrefixReg.MatchString(keyword) {
+				} else if statusPrefixReg.MatchString(keyword) { // status:200
+					isSpecialKeyword = true
 					var matches = statusPrefixReg.FindStringSubmatch(keyword)
 					query.Attr("status", matches[1])
-					// TODO 处理剩余的关键词
-				} else {
+				} else if requestPathReg.MatchString(keyword) {
+					isSpecialKeyword = true
+					var matches = requestPathReg.FindStringSubmatch(keyword)
+					query.Where("JSON_EXTRACT(content, '$.requestPath')=:keyword").
+						Param("keyword", matches[1])
+				} else if protoReg.MatchString(keyword) {
+					isSpecialKeyword = true
+					var matches = protoReg.FindStringSubmatch(keyword)
+					query.Where("JSON_EXTRACT(content, '$.proto')=:keyword").
+						Param("keyword", strings.ToUpper(matches[1]))
+				} else if schemeReg.MatchString(keyword) {
+					isSpecialKeyword = true
+					var matches = schemeReg.FindStringSubmatch(keyword)
+					query.Where("JSON_EXTRACT(content, '$.scheme')=:keyword").
+						Param("keyword", strings.ToLower(matches[1]))
+				} else if urlReg.MatchString(keyword) { // https://xxx/yyy
+					u, err := url.Parse(keyword)
+					if err == nil {
+						isSpecialKeyword = true
+						query.Attr("domain", u.Host)
+						query.Where("JSON_EXTRACT(content, '$.requestURI') LIKE :keyword").
+							Param("keyword", dbutils.QuoteLikePrefix("\""+u.RequestURI()))
+					}
+				}
+				if !isSpecialKeyword {
 					if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
 						keyword = keyword[3:]
 					}
@@ -530,7 +590,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 					}
 
 					query.Where("("+where+")").
-						Param("keyword", "%"+keyword+"%")
+						Param("keyword", dbutils.QuoteLike(keyword))
 					if useOriginKeyword {
 						query.Param("originKeyword", keyword)
 					}
@@ -574,17 +634,17 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 				Limit(size).
 				FindAll()
 			if err != nil {
-				logs.Println("[DB_NODE]" + err.Error())
+				remotelogs.Println("DB_NODE", err.Error())
 				return
 			}
 
 			locker.Lock()
 			for _, one := range ones {
-				accessLog := one.(*HTTPAccessLog)
+				var accessLog = one.(*HTTPAccessLog)
 				result = append(result, accessLog)
 			}
 			locker.Unlock()
-		}(tableQuery)
+		}(tableQuery, keyword)
 	}
 	wg.Wait()
 

internal/db/models/http_access_log_dao_test.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
@@ -53,7 +54,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, "", 10, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -80,7 +81,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
 	times := 0 // 防止循环次数太多
 	for {
 		before := time.Now()
-		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, lastRequestId, 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 		cost := time.Since(before).Seconds()
 		if err != nil {
 			t.Fatal(err)
@@ -111,7 +112,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
 	}
 
 	before := time.Now()
-	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
+	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
 	cost := time.Since(before).Seconds()
 	if err != nil {
 		t.Fatal(err)
@@ -136,7 +137,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
 	times := 0 // 防止循环次数太多
 	for {
 		before := time.Now()
-		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 		cost := time.Since(before).Seconds()
 		if err != nil {
 			t.Fatal(err)
@@ -157,3 +158,13 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkHTTPAccessLogDAO_JSONEncode(b *testing.B) {
+	var accessLog = &pb.HTTPAccessLog{
+		RequestPath: "/hello/world",
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, _ = json.Marshal(accessLog)
+	}
+}

internal/db/models/http_access_log_manager.go

@@ -15,6 +15,7 @@ import (
 )
 
 // 访问日志的两个表格形式
+// 括号位置需要固定，会用来读取日期和分区
 var accessLogTableMainReg = regexp.MustCompile(`_(\d{8})$`)
 var accessLogTablePartialReg = regexp.MustCompile(`_(\d{8})_(\d{4})$`)
 
@@ -38,7 +39,7 @@ func (this *HTTPAccessLogManager) FindTableNames(db *dbs.DB, day string) ([]stri
 
 	// 需要防止用户设置了表名自动小写
 	for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
-		ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
+		ones, columnNames, err := db.FindPreparedOnes(`SHOW TABLES LIKE '` + prefix + `'`)
 		if err != nil {
 			return nil, errors.New("query table names error: " + err.Error())
 		}
@@ -77,9 +78,15 @@ func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAcc
 	var results = []*httpAccessLogDefinition{}
 	var tableNames = []string{}
 
+	config, err := db.Config()
+	if err != nil {
+		return nil, err
+	}
+	var cachePrefix = config.Dsn
+
 	// 需要防止用户设置了表名自动小写
 	for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
-		ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
+		ones, columnNames, err := db.FindPreparedOnes(`SHOW TABLES LIKE '` + prefix + `'`)
 		if err != nil {
 			return nil, errors.New("query table names error: " + err.Error())
 		}
@@ -96,17 +103,32 @@ func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAcc
 			if accessLogTableMainReg.MatchString(tableName) {
 				tableNames = append(tableNames, tableName)
 
-				hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, tableName)
-				if err != nil {
-					return nil, err
-				}
+				// 查找已有的表格信息，避免SHOW FIELDS
+				var tableDay = tableName[strings.LastIndex(tableName, "_")+1:]
+				var cacheKey = this.composeTableCacheKey(cachePrefix, tableDay)
+				this.locker.Lock()
+				currentTableDef, ok := this.currentTableMapping[cacheKey]
+				this.locker.Unlock()
+				if ok {
+					results = append(results, &httpAccessLogDefinition{
+						Name:          tableName,
+						HasRemoteAddr: currentTableDef.HasRemoteAddr,
+						HasDomain:     currentTableDef.HasDomain,
+						Exists:        true,
+					})
+				} else {
+					hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, tableName)
+					if err != nil {
+						return nil, err
+					}
 
-				results = append(results, &httpAccessLogDefinition{
-					Name:          tableName,
-					HasRemoteAddr: hasRemoteAddrField,
-					HasDomain:     hasDomainField,
-					Exists:        true,
-				})
+					results = append(results, &httpAccessLogDefinition{
+						Name:          tableName,
+						HasRemoteAddr: hasRemoteAddrField,
+						HasDomain:     hasDomainField,
+						Exists:        true,
+					})
+				}
 			} else if accessLogTablePartialReg.MatchString(tableName) {
 				tableNames = append(tableNames, tableName)
 
@@ -128,11 +150,55 @@ func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAcc
 	return results, nil
 }
 
-// FindTable 根据日期获取表名
+func (this *HTTPAccessLogManager) FindPartitionTable(db *dbs.DB, day string, partition int32) (*httpAccessLogDefinition, error) {
+	var tableNames []string
+	if partition < 0 {
+		tableList, err := this.FindTables(db, day)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(tableList) > 0 {
+			return tableList[len(tableList)-1], nil
+		}
+
+		return &httpAccessLogDefinition{
+			Name:          "",
+			HasRemoteAddr: false,
+			HasDomain:     false,
+			Exists:        false,
+		}, nil
+	} else if partition == 0 {
+		tableNames = []string{"edgeHTTPAccessLogs_" + day, "edgehttpaccesslogs_" + day}
+	} else {
+		tableNames = []string{"edgeHTTPAccessLogs_" + day + "_" + fmt.Sprintf("%04d", partition), "edgehttpaccesslogs_" + day + "_" + fmt.Sprintf("%04d", partition)}
+	}
+	for _, tableName := range tableNames {
+		hasRemoteField, hasDomainField, err := this.checkTableFields(db, tableName)
+		if err != nil {
+			continue
+		}
+		return &httpAccessLogDefinition{
+			Name:          tableName,
+			HasRemoteAddr: hasRemoteField,
+			HasDomain:     hasDomainField,
+			Exists:        true,
+		}, nil
+	}
+
+	return &httpAccessLogDefinition{
+		Name:          "",
+		HasRemoteAddr: false,
+		HasDomain:     false,
+		Exists:        false,
+	}, nil
+}
+
+// FindLastTable 根据日期获取上一个可以使用的表名
 // 表名组成
 //   - PREFIX_DAY
 //   - PREFIX_DAY_0001
-func (this *HTTPAccessLogManager) FindTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
+func (this *HTTPAccessLogManager) FindLastTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
 	this.locker.Lock()
 	defer this.locker.Unlock()
 
@@ -197,6 +263,60 @@ func (this *HTTPAccessLogManager) ResetTable(db *dbs.DB, day string) {
 	delete(this.currentTableMapping, this.composeTableCacheKey(config.Dsn, day))
 }
 
+// TablePartition 从表名中获取分区
+func (this *HTTPAccessLogManager) TablePartition(tableName string) (partition int32) {
+	if accessLogTablePartialReg.MatchString(tableName) {
+		return types.Int32(accessLogTablePartialReg.FindStringSubmatch(tableName)[2])
+	}
+
+	return 0
+}
+
+// FindLatestPartition 读取最后一个分区
+func (this *HTTPAccessLogManager) FindLatestPartition(day string) (int32, error) {
+	var dbList = AllAccessLogDBs()
+	if len(dbList) == 0 {
+		return 0, errors.New("no valid database")
+	}
+
+	var partitions = []int32{}
+	var locker sync.Mutex
+
+	var wg = sync.WaitGroup{}
+	wg.Add(len(dbList))
+
+	var lastErr error
+	for _, db := range dbList {
+		go func(db *dbs.DB) {
+			defer wg.Done()
+
+			names, err := this.FindTableNames(db, day)
+			if err != nil {
+				lastErr = err
+			}
+			for _, name := range names {
+				var partition = this.TablePartition(name)
+				locker.Lock()
+				if !lists.Contains(partitions, partition) {
+					partitions = append(partitions, partition)
+				}
+				locker.Unlock()
+			}
+		}(db)
+	}
+	wg.Wait()
+
+	if lastErr != nil {
+		return 0, lastErr
+	}
+
+	if len(partitions) == 0 {
+		return 0, nil
+	}
+
+	return partitions[len(partitions)-1], nil
+}
+
 // 查找某个表格
 func (this *HTTPAccessLogManager) findTableWithoutCache(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
 	tableNames, err := this.FindTableNames(db, day)
@@ -296,7 +416,7 @@ func (this *HTTPAccessLogManager) findTableWithoutCache(db *dbs.DB, day string,
 
 // TODO 考虑缓存检查结果
 func (this *HTTPAccessLogManager) checkTableFields(db *dbs.DB, tableName string) (hasRemoteAddrField bool, hasDomainField bool, err error) {
-	fields, _, err := db.FindOnes("SHOW FIELDS FROM " + tableName)
+	fields, _, err := db.FindPreparedOnes("SHOW FIELDS FROM " + tableName)
 	if err != nil {
 		return false, false, err
 	}

internal/db/models/http_access_log_manager_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"testing"
 	"time"
 )
@@ -30,6 +31,9 @@ func TestNewHTTPAccessLogManager(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	var manager = models.SharedHTTPAccessLogManager
 	err = manager.CreateTable(db, "accessLog_1")
@@ -58,6 +62,9 @@ func TestHTTPAccessLogManager_FindTableNames(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	for i := 0; i < 3; i++ {
 		var before = time.Now()
@@ -74,7 +81,6 @@ func TestHTTPAccessLogManager_FindTableNames(t *testing.T) {
 	}
 }
 
-
 func TestHTTPAccessLogManager_FindTables(t *testing.T) {
 	var config = &dbs.DBConfig{
 		Driver: "mysql",
@@ -95,6 +101,9 @@ func TestHTTPAccessLogManager_FindTables(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	for i := 0; i < 3; i++ {
 		var before = time.Now()
@@ -111,7 +120,7 @@ func TestHTTPAccessLogManager_FindTables(t *testing.T) {
 	}
 }
 
-func TestHTTPAccessLogManager_FindTable(t *testing.T) {
+func TestHTTPAccessLogManager_FindLastTable(t *testing.T) {
 	var config = &dbs.DBConfig{
 		Driver: "mysql",
 		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
@@ -131,10 +140,13 @@ func TestHTTPAccessLogManager_FindTable(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	for i := 0; i < 3; i++ {
 		var before = time.Now()
-		tableDef, err := models.SharedHTTPAccessLogManager.FindTable(db, "20220306", false)
+		tableDef, err := models.SharedHTTPAccessLogManager.FindLastTable(db, "20220306", false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -146,3 +158,32 @@ func TestHTTPAccessLogManager_FindTable(t *testing.T) {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}
 }
+
+func TestHTTPAccessLogManager_FindPartitionTable(t *testing.T) {
+	var config = &dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+		Connections: struct {
+			Pool         int           `yaml:"pool"`
+			Max          int           `yaml:"max"`
+			Life         string        `yaml:"life"`
+			LifeDuration time.Duration `yaml:",omitempty"`
+		}{},
+		Models: struct {
+			Package string `yaml:"package"`
+		}{},
+	}
+
+	db, err := dbs.NewInstanceFromConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = db.Close()
+	}()
+
+	t.Log(models.SharedHTTPAccessLogManager.FindPartitionTable(db, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), -1))
+	t.Log(models.SharedHTTPAccessLogManager.FindPartitionTable(db, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), 0))
+	t.Log(models.SharedHTTPAccessLogManager.FindPartitionTable(db, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), 1))
+}

internal/db/models/http_access_log_policy_dao.go

@@ -1,13 +1,10 @@
 package models
 
 import (
-	"bytes"
-	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
-	"github.com/iwind/TeaGo/maps"
 )
 
 const (
@@ -109,7 +106,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
 }
 
 // CreatePolicy 创建策略
-func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool) (policyId int64, err error) {
+func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool) (policyId int64, err error) {
 	var op = NewHTTPAccessLogPolicyOperator()
 	op.Name = name
 	op.Type = policyType
@@ -121,12 +118,13 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
 	}
 	op.IsPublic = isPublic
 	op.IsOn = true
+	op.FirewallOnly = firewallOnly
 	op.State = HTTPAccessLogPolicyStateEnabled
 	return this.SaveInt64(tx, op)
 }
 
 // UpdatePolicy 修改策略
-func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, isOn bool) error {
+func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, isOn bool) error {
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
@@ -140,7 +138,6 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
 	if oldOne == nil {
 		return nil
 	}
-	var oldPolicy = oldOne.(*HTTPAccessLogPolicy)
 
 	var op = NewHTTPAccessLogPolicyOperator()
 	op.Id = policyId
@@ -156,22 +153,11 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
 		op.Conds = "{}"
 	}
 
-	// 版本号
-	if len(oldPolicy.Options) == 0 || len(optionsJSON) == 0 {
-		op.Version = dbs.SQL("version+1")
-	} else {
-		var m1 = maps.Map{}
-		_ = json.Unmarshal(oldPolicy.Options, &m1)
-
-		var m2 = maps.Map{}
-		_ = json.Unmarshal(optionsJSON, &m2)
-
-		if bytes.Compare(m1.AsJSON(), m2.AsJSON()) != 0 {
-			op.Version = dbs.SQL("version+1")
-		}
-	}
+	// 版本号总是加1
+	op.Version = dbs.SQL("version+1")
 
 	op.IsPublic = isPublic
+	op.FirewallOnly = firewallOnly
 	op.IsOn = isOn
 	return this.Save(tx, op)
 }

internal/db/models/http_access_log_policy_model.go

@@ -4,35 +4,37 @@ import "github.com/iwind/TeaGo/dbs"
 
 // HTTPAccessLogPolicy 访问日志策略
 type HTTPAccessLogPolicy struct {
-	Id         uint32   `field:"id"`         // ID
-	TemplateId uint32   `field:"templateId"` // 模版ID
-	AdminId    uint32   `field:"adminId"`    // 管理员ID
-	UserId     uint32   `field:"userId"`     // 用户ID
-	State      uint8    `field:"state"`      // 状态
-	CreatedAt  uint64   `field:"createdAt"`  // 创建时间
-	Name       string   `field:"name"`       // 名称
-	IsOn       bool     `field:"isOn"`       // 是否启用
-	Type       string   `field:"type"`       // 存储类型
-	Options    dbs.JSON `field:"options"`    // 存储选项
-	Conds      dbs.JSON `field:"conds"`      // 请求条件
-	IsPublic   bool     `field:"isPublic"`   // 是否为公用
-	Version    uint32   `field:"version"`    // 版本号
+	Id           uint32   `field:"id"`           // ID
+	TemplateId   uint32   `field:"templateId"`   // 模版ID
+	AdminId      uint32   `field:"adminId"`      // 管理员ID
+	UserId       uint32   `field:"userId"`       // 用户ID
+	State        uint8    `field:"state"`        // 状态
+	CreatedAt    uint64   `field:"createdAt"`    // 创建时间
+	Name         string   `field:"name"`         // 名称
+	IsOn         bool     `field:"isOn"`         // 是否启用
+	Type         string   `field:"type"`         // 存储类型
+	Options      dbs.JSON `field:"options"`      // 存储选项
+	Conds        dbs.JSON `field:"conds"`        // 请求条件
+	IsPublic     bool     `field:"isPublic"`     // 是否为公用
+	FirewallOnly uint8    `field:"firewallOnly"` // 是否只记录防火墙相关
+	Version      uint32   `field:"version"`      // 版本号
 }
 
 type HTTPAccessLogPolicyOperator struct {
-	Id         interface{} // ID
-	TemplateId interface{} // 模版ID
-	AdminId    interface{} // 管理员ID
-	UserId     interface{} // 用户ID
-	State      interface{} // 状态
-	CreatedAt  interface{} // 创建时间
-	Name       interface{} // 名称
-	IsOn       interface{} // 是否启用
-	Type       interface{} // 存储类型
-	Options    interface{} // 存储选项
-	Conds      interface{} // 请求条件
-	IsPublic   interface{} // 是否为公用
-	Version    interface{} // 版本号
+	Id           interface{} // ID
+	TemplateId   interface{} // 模版ID
+	AdminId      interface{} // 管理员ID
+	UserId       interface{} // 用户ID
+	State        interface{} // 状态
+	CreatedAt    interface{} // 创建时间
+	Name         interface{} // 名称
+	IsOn         interface{} // 是否启用
+	Type         interface{} // 存储类型
+	Options      interface{} // 存储选项
+	Conds        interface{} // 请求条件
+	IsPublic     interface{} // 是否为公用
+	FirewallOnly interface{} // 是否只记录防火墙相关
+	Version      interface{} // 版本号
 }
 
 func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {

internal/db/models/http_auth_policy_dao.go

@@ -69,7 +69,7 @@ func (this *HTTPAuthPolicyDAO) FindEnabledHTTPAuthPolicy(tx *dbs.Tx, id int64) (
 
 // CreateHTTPAuthPolicy 创建策略
 func (this *HTTPAuthPolicyDAO) CreateHTTPAuthPolicy(tx *dbs.Tx, name string, methodType string, paramsJSON []byte) (int64, error) {
-	op := NewHTTPAuthPolicyOperator()
+	var op = NewHTTPAuthPolicyOperator()
 	op.Name = name
 	op.Type = methodType
 	op.Params = paramsJSON
@@ -83,7 +83,7 @@ func (this *HTTPAuthPolicyDAO) UpdateHTTPAuthPolicy(tx *dbs.Tx, policyId int64,
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPAuthPolicyOperator()
+	var op = NewHTTPAuthPolicyOperator()
 	op.Id = policyId
 	op.Name = name
 	op.Params = paramsJSON

internal/db/models/http_brotli_policy_dao.go

@@ -113,7 +113,7 @@ func (this *HTTPBrotliPolicyDAO) ComposeBrotliConfig(tx *dbs.Tx, policyId int64)
 
 // CreatePolicy 创建策略
 func (this *HTTPBrotliPolicyDAO) CreatePolicy(tx *dbs.Tx, level int, minLengthJSON []byte, maxLengthJSON []byte, condsJSON []byte) (int64, error) {
-	op := NewHTTPBrotliPolicyOperator()
+	var op = NewHTTPBrotliPolicyOperator()
 	op.State = HTTPBrotliPolicyStateEnabled
 	op.IsOn = true
 	op.Level = level
@@ -138,7 +138,7 @@ func (this *HTTPBrotliPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, level
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPBrotliPolicyOperator()
+	var op = NewHTTPBrotliPolicyOperator()
 	op.Id = policyId
 	op.Level = level
 	if len(minLengthJSON) > 0 {

internal/db/models/http_cache_policy_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
@@ -96,7 +97,7 @@ func (this *HTTPCachePolicyDAO) FindAllEnabledCachePolicies(tx *dbs.Tx) (result
 
 // CreateCachePolicy 创建缓存策略
 func (this *HTTPCachePolicyDAO) CreateCachePolicy(tx *dbs.Tx, isOn bool, name string, description string, capacityJSON []byte, maxKeys int64, maxSizeJSON []byte, storageType string, storageOptionsJSON []byte, syncCompressionCache bool) (int64, error) {
-	op := NewHTTPCachePolicyOperator()
+	var op = NewHTTPCachePolicyOperator()
 	op.State = HTTPCachePolicyStateEnabled
 	op.IsOn = isOn
 	op.Name = name
@@ -208,7 +209,7 @@ func (this *HTTPCachePolicyDAO) UpdateCachePolicy(tx *dbs.Tx, policyId int64, is
 		return errors.New("invalid policyId")
 	}
 
-	op := NewHTTPCachePolicyOperator()
+	var op = NewHTTPCachePolicyOperator()
 	op.Id = policyId
 	op.IsOn = isOn
 	op.Name = name
@@ -318,7 +319,7 @@ func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clu
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(storageType) > 0 {
 		query.Attr("type", storageType)
@@ -336,7 +337,7 @@ func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, cluster
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(storageType) > 0 {
 		query.Attr("type", storageType)

internal/db/models/http_cache_task_dao.go

@@ -0,0 +1,254 @@
+package models
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+const (
+	HTTPCacheTaskStateEnabled  = 1 // 已启用
+	HTTPCacheTaskStateDisabled = 0 // 已禁用
+)
+
+type HTTPCacheTaskType = string
+
+const (
+	HTTPCacheTaskTypePurge HTTPCacheTaskType = "purge"
+	HTTPCacheTaskTypeFetch HTTPCacheTaskType = "fetch"
+)
+
+type HTTPCacheTaskDAO dbs.DAO
+
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
+		goman.New(func() {
+			for range ticker.C {
+				err := SharedHTTPCacheTaskDAO.Clean(nil, 30) // 只保留N天
+				if err != nil {
+					remotelogs.Error("HTTPCacheTaskDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		})
+	})
+}
+
+func NewHTTPCacheTaskDAO() *HTTPCacheTaskDAO {
+	return dbs.NewDAO(&HTTPCacheTaskDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeHTTPCacheTasks",
+			Model:  new(HTTPCacheTask),
+			PkName: "id",
+		},
+	}).(*HTTPCacheTaskDAO)
+}
+
+var SharedHTTPCacheTaskDAO *HTTPCacheTaskDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedHTTPCacheTaskDAO = NewHTTPCacheTaskDAO()
+	})
+}
+
+// EnableHTTPCacheTask 启用条目
+func (this *HTTPCacheTaskDAO) EnableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
+	_, err := this.Query(tx).
+		Pk(taskId).
+		Set("state", HTTPCacheTaskStateEnabled).
+		Update()
+	return err
+}
+
+// DisableHTTPCacheTask 禁用条目
+func (this *HTTPCacheTaskDAO) DisableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
+	_, err := this.Query(tx).
+		Pk(taskId).
+		Set("state", HTTPCacheTaskStateDisabled).
+		Update()
+	if err != nil {
+		return err
+	}
+	return this.NotifyChange(tx, taskId)
+}
+
+// FindEnabledHTTPCacheTask 查找启用中的条目
+func (this *HTTPCacheTaskDAO) FindEnabledHTTPCacheTask(tx *dbs.Tx, taskId int64) (*HTTPCacheTask, error) {
+	result, err := this.Query(tx).
+		Pk(taskId).
+		Attr("state", HTTPCacheTaskStateEnabled).
+		Find()
+	if result == nil {
+		return nil, err
+	}
+	return result.(*HTTPCacheTask), err
+}
+
+// CreateTask 创建任务
+func (this *HTTPCacheTaskDAO) CreateTask(tx *dbs.Tx, userId int64, taskType HTTPCacheTaskType, keyType string, description string) (int64, error) {
+	var op = NewHTTPCacheTaskOperator()
+	op.UserId = userId
+	op.Type = taskType
+	op.KeyType = keyType
+	op.IsOk = false
+	op.IsDone = false
+	op.IsReady = false
+	op.Description = description
+	op.Day = timeutil.Format("Ymd")
+	op.State = HTTPCacheTaskStateEnabled
+	taskId, err := this.SaveInt64(tx, op)
+	if err != nil {
+		return 0, err
+	}
+
+	err = this.NotifyChange(tx, taskId)
+	if err != nil {
+		return 0, err
+	}
+	return taskId, nil
+}
+
+// ResetTask 重置服务状态
+func (this *HTTPCacheTaskDAO) ResetTask(tx *dbs.Tx, taskId int64) error {
+	if taskId <= 0 {
+		return errors.New("invalid 'taskId'")
+	}
+
+	var op = NewHTTPCacheTaskOperator()
+	op.Id = taskId
+	op.IsOk = false
+	op.IsDone = false
+	op.DoneAt = 0
+	return this.Save(tx, op)
+}
+
+// UpdateTaskReady 设置任务为已准备
+func (this *HTTPCacheTaskDAO) UpdateTaskReady(tx *dbs.Tx, taskId int64) error {
+	return this.Query(tx).
+		Pk(taskId).
+		Set("isReady", true).
+		UpdateQuickly()
+}
+
+// CountTasks 查询所有任务数量
+func (this *HTTPCacheTaskDAO) CountTasks(tx *dbs.Tx, userId int64) (int64, error) {
+	var query = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isReady", true)
+	if userId > 0 {
+		query.Attr("userId", userId)
+	}
+	return query.Count()
+}
+
+// CountDoingTasks 查询正在执行的任务数量
+func (this *HTTPCacheTaskDAO) CountDoingTasks(tx *dbs.Tx, userId int64) (int64, error) {
+	var query = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isReady", true).
+		Attr("isDone", false)
+	if userId > 0 {
+		query.Attr("userId", userId)
+	}
+
+	return query.Count()
+}
+
+// ListTasks 列出单页任务
+func (this *HTTPCacheTaskDAO) ListTasks(tx *dbs.Tx, userId int64, offset int64, size int64) (result []*HTTPCacheTask, err error) {
+	var query = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isReady", true)
+	if userId > 0 {
+		query.Attr("userId", userId)
+	}
+	_, err = query.
+		Offset(offset).
+		Limit(size).
+		Slice(&result).
+		DescPk().
+		FindAll()
+	return
+}
+
+// ListDoingTasks 列出需要执行的任务
+func (this *HTTPCacheTaskDAO) ListDoingTasks(tx *dbs.Tx, size int64) (result []*HTTPCacheTask, err error) {
+	_, err = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isDone", false).
+		Attr("isReady", true).
+		Limit(size).
+		AscPk(). // 按照先创建先执行的原则
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// UpdateTaskStatus 标记任务已完成
+func (this *HTTPCacheTaskDAO) UpdateTaskStatus(tx *dbs.Tx, taskId int64, isDone bool, isOk bool) error {
+	if taskId <= 0 {
+		return errors.New("invalid taskId '" + types.String(taskId) + "'")
+	}
+	var op = NewHTTPCacheTaskOperator()
+	op.Id = taskId
+	op.IsDone = isDone
+	op.IsOk = isOk
+
+	if isDone {
+		op.DoneAt = time.Now().Unix()
+	}
+
+	return this.Save(tx, op)
+}
+
+// CheckUserTask 检查用户任务
+func (this *HTTPCacheTaskDAO) CheckUserTask(tx *dbs.Tx, userId int64, taskId int64) error {
+	b, err := this.Query(tx).
+		Pk(taskId).
+		Attr("userId", userId).
+		State(HTTPCacheTaskStateEnabled).
+		Exist()
+	if err != nil {
+		return err
+	}
+	if !b {
+		return ErrNotFound
+	}
+	return nil
+}
+
+// Clean 清理以往的任务
+func (this *HTTPCacheTaskDAO) Clean(tx *dbs.Tx, days int) error {
+	if days <= 0 {
+		days = 30
+	}
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
+
+	// 删除Key
+	err := SharedHTTPCacheTaskKeyDAO.Clean(tx, days)
+	if err != nil {
+		return err
+	}
+
+	// 删除任务
+	_, err = this.Query(tx).
+		Lte("day", day).
+		Delete()
+	return err
+}
+
+// NotifyChange 发送通知
+func (this *HTTPCacheTaskDAO) NotifyChange(tx *dbs.Tx, taskId int64) error {
+	// TODO
+	return nil
+}

internal/db/models/http_cache_task_dao_test.go

@@ -0,0 +1,19 @@
+package models_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+)
+
+func TestHTTPCacheTaskDAO_Clean(t *testing.T) {
+	dbs.NotifyReady()
+
+	err := models.SharedHTTPCacheTaskDAO.Clean(nil, 30)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

internal/db/models/http_cache_task_key_dao.go

@@ -0,0 +1,218 @@
+package models
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+type HTTPCacheTaskKeyDAO dbs.DAO
+
+func NewHTTPCacheTaskKeyDAO() *HTTPCacheTaskKeyDAO {
+	return dbs.NewDAO(&HTTPCacheTaskKeyDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeHTTPCacheTaskKeys",
+			Model:  new(HTTPCacheTaskKey),
+			PkName: "id",
+		},
+	}).(*HTTPCacheTaskKeyDAO)
+}
+
+var SharedHTTPCacheTaskKeyDAO *HTTPCacheTaskKeyDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedHTTPCacheTaskKeyDAO = NewHTTPCacheTaskKeyDAO()
+	})
+}
+
+// CreateKey 创建Key
+// 参数：
+//   - clusterId 集群ID
+//   - nodeMapJSON 集群下节点映射，格式类似于 `{ "节点1":true, ... }`
+func (this *HTTPCacheTaskKeyDAO) CreateKey(tx *dbs.Tx, taskId int64, key string, taskType HTTPCacheTaskType, keyType string, clusterId int64) (int64, error) {
+	var op = NewHTTPCacheTaskKeyOperator()
+	op.TaskId = taskId
+	op.Key = key
+	op.Type = taskType
+	op.KeyType = keyType
+	op.ClusterId = clusterId
+
+	op.Nodes = "{}"
+	op.Errors = "{}"
+
+	return this.SaveInt64(tx, op)
+}
+
+// UpdateKeyStatus 修改Key状态
+func (this *HTTPCacheTaskKeyDAO) UpdateKeyStatus(tx *dbs.Tx, keyId int64, nodeId int64, errString string, nodesJSON []byte) error {
+	if keyId <= 0 {
+		return errors.New("invalid 'keyId'")
+	}
+
+	if len(nodesJSON) == 0 {
+		nodesJSON = []byte("{}")
+	}
+
+	taskId, err := this.Query(tx).
+		Pk(keyId).
+		Result("taskId").
+		FindInt64Col(0)
+	if err != nil {
+		return err
+	}
+
+	var jsonPath = "$.\"" + types.String(nodeId) + "\""
+
+	var query = this.Query(tx).
+		Pk(keyId).
+		Set("nodes", dbs.SQL("JSON_SET(nodes, :jsonPath1, true)")).
+		Param("jsonPath1", jsonPath)
+
+	if len(errString) > 0 {
+		query.Set("errors", dbs.SQL("JSON_SET(errors, :jsonPath2, :jsonValue2)")).
+			Param("jsonPath2", jsonPath).
+			Param("jsonValue2", errString)
+	} else {
+		query.Set("errors", dbs.SQL("JSON_REMOVE(errors, :jsonPath2)")).
+			Param("jsonPath2", jsonPath)
+	}
+
+	err = query.
+		UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	// 检查是否已完成
+	isDone, err := this.Query(tx).
+		Pk(keyId).
+		Where("JSON_CONTAINS(nodes, :nodesJSON)").
+		Param("nodesJSON", nodesJSON).
+		Exist()
+	if err != nil {
+		return err
+	}
+
+	if isDone {
+		err = this.Query(tx).
+			Pk(keyId).
+			Set("isDone", isDone).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+
+		// 检查任务是否已经完成
+		taskIsNotDone, err := this.Query(tx).
+			Attr("taskId", taskId).
+			Attr("isDone", false).
+			Exist()
+		if err != nil {
+			return err
+		}
+		var taskIsDone = !taskIsNotDone
+		var hasErrors = true
+		if taskIsDone {
+			// 已经完成，是否有错误
+			hasErrors, err = this.Query(tx).
+				Attr("taskId", taskId).
+				Where("JSON_LENGTH(errors)>0").
+				Exist()
+			if err != nil {
+				return err
+			}
+		}
+		err = SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, taskIsDone, !hasErrors)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// FindAllTaskKeys 查询某个任务下的所有Key
+func (this *HTTPCacheTaskKeyDAO) FindAllTaskKeys(tx *dbs.Tx, taskId int64) (result []*HTTPCacheTaskKey, err error) {
+	_, err = this.Query(tx).
+		Attr("taskId", taskId).
+		AscPk().
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// FindDoingTaskKeys 查询要执行的任务
+func (this *HTTPCacheTaskKeyDAO) FindDoingTaskKeys(tx *dbs.Tx, nodeId int64, size int64) (result []*HTTPCacheTaskKey, err error) {
+	// 集群ID
+	clusterIds, err := SharedNodeDAO.FindEnabledAndOnNodeClusterIds(tx, nodeId)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(clusterIds) == 0 {
+		return nil, nil
+	}
+
+	_, err = this.Query(tx).
+		Attr("clusterId", clusterIds).
+		Attr("isDone", false).
+		Where("NOT JSON_CONTAINS_PATH(nodes, 'one', :jsonPath1)").
+		Param("jsonPath1", "$.\""+types.String(nodeId)+"\"").
+		Where("taskId IN (SELECT id FROM " + SharedHTTPCacheTaskDAO.Table + " WHERE state=1 AND isReady=1 AND isDone=0)").
+		Limit(size).
+		AscPk().
+		Reuse(false).
+		Slice(&result).
+		FindAll()
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+// ResetCacheKeysWithTaskId 重置任务下的Key状态
+func (this *HTTPCacheTaskKeyDAO) ResetCacheKeysWithTaskId(tx *dbs.Tx, taskId int64) error {
+	return this.Query(tx).
+		Attr("taskId", taskId).
+		Set("isDone", false).
+		Set("nodes", "{}").
+		Set("errors", "{}").
+		UpdateQuickly()
+}
+
+// CountUserTasksInDay 读取某个用户当前数量
+// day YYYYMMDD
+func (this *HTTPCacheTaskKeyDAO) CountUserTasksInDay(tx *dbs.Tx, userId int64, day string, taskType HTTPCacheTaskType) (int64, error) {
+	if userId <= 0 {
+		return 0, nil
+	}
+
+	// 这里需要包含已删除的
+	return this.Query(tx).
+		Where("taskId IN (SELECT id FROM "+SharedHTTPCacheTaskDAO.Table+" WHERE userId=:userId AND day=:day AND type=:type)").
+		Param("userId", userId).
+		Param("day", day).
+		Param("type", taskType).
+		Count()
+}
+
+// Clean 清理以往的任务
+func (this *HTTPCacheTaskKeyDAO) Clean(tx *dbs.Tx, days int) error {
+	if days <= 0 {
+		days = 30
+	}
+
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Where("taskId IN (SELECT id FROM "+SharedHTTPCacheTaskDAO.Table+" WHERE day<=:day)").
+		Param("day", day).
+		Delete()
+	return err
+}

internal/db/models/http_cache_task_key_dao_test.go

@@ -0,0 +1,54 @@
+package models_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"testing"
+)
+
+func TestHTTPCacheTaskKeyDAO_CreateKey(t *testing.T) {
+	var dao = models.NewHTTPCacheTaskKeyDAO()
+	var tx *dbs.Tx
+	_, err := dao.CreateKey(tx, 1, "a", "purge", "key", 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestHTTPCacheTaskKeyDAO_UpdateKeyStatus(t *testing.T) {
+	dbs.NotifyReady()
+
+	var dao = models.NewHTTPCacheTaskKeyDAO()
+	var tx *dbs.Tx
+	var errString = "" // "this is error"
+	err := dao.UpdateKeyStatus(tx, 3, 1, errString, []byte(`{"1":true}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestHTTPCacheTaskKeyDAO_CountUserTasksInDay(t *testing.T) {
+	dbs.NotifyReady()
+
+	var dao = models.NewHTTPCacheTaskKeyDAO()
+	var tx *dbs.Tx
+	{
+		count, err := dao.CountUserTasksInDay(tx, 1, timeutil.Format("Ymd"), models.HTTPCacheTaskTypePurge)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("count:", count)
+	}
+	{
+		count, err := dao.CountUserTasksInDay(tx, 1, timeutil.Format("Ymd"), models.HTTPCacheTaskTypeFetch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("count:", count)
+	}
+}

internal/db/models/http_cache_task_key_model.go

@@ -0,0 +1,32 @@
+package models
+
+import "github.com/iwind/TeaGo/dbs"
+
+// HTTPCacheTaskKey 缓存任务Key
+type HTTPCacheTaskKey struct {
+	Id        uint64   `field:"id"`        // ID
+	TaskId    uint64   `field:"taskId"`    // 任务ID
+	Key       string   `field:"key"`       // Key
+	KeyType   string   `field:"keyType"`   // Key类型：key|prefix
+	Type      string   `field:"type"`      // 操作类型
+	ClusterId uint32   `field:"clusterId"` // 集群ID
+	Nodes     dbs.JSON `field:"nodes"`     // 节点
+	Errors    dbs.JSON `field:"errors"`    // 错误信息
+	IsDone    bool     `field:"isDone"`    // 是否已完成
+}
+
+type HTTPCacheTaskKeyOperator struct {
+	Id        interface{} // ID
+	TaskId    interface{} // 任务ID
+	Key       interface{} // Key
+	KeyType   interface{} // Key类型：key|prefix
+	Type      interface{} // 操作类型
+	ClusterId interface{} // 集群ID
+	Nodes     interface{} // 节点
+	Errors    interface{} // 错误信息
+	IsDone    interface{} // 是否已完成
+}
+
+func NewHTTPCacheTaskKeyOperator() *HTTPCacheTaskKeyOperator {
+	return &HTTPCacheTaskKeyOperator{}
+}

internal/db/models/http_cache_task_key_model_ext.go

@@ -0,0 +1,20 @@
+package models
+
+import "encoding/json"
+
+// DecodeNodes 解析已完成节点信息
+func (this *HTTPCacheTaskKey) DecodeNodes() map[string]bool {
+	var result = map[string]bool{}
+	var nodesJSON = this.Nodes
+	if IsNull(nodesJSON) {
+		return result
+	}
+
+	err := json.Unmarshal(nodesJSON, &result)
+	if err != nil {
+		// ignore error
+		return result
+	}
+
+	return result
+}

internal/db/models/http_cache_task_model.go

@@ -0,0 +1,36 @@
+package models
+
+// HTTPCacheTask 缓存相关任务
+type HTTPCacheTask struct {
+	Id          uint64 `field:"id"`          // ID
+	UserId      uint32 `field:"userId"`      // 用户ID
+	Type        string `field:"type"`        // 任务类型：purge|fetch
+	KeyType     string `field:"keyType"`     // Key类型
+	State       uint8  `field:"state"`       // 状态
+	CreatedAt   uint64 `field:"createdAt"`   // 创建时间
+	DoneAt      uint64 `field:"doneAt"`      // 完成时间
+	Day         string `field:"day"`         // 创建日期YYYYMMDD
+	IsDone      bool   `field:"isDone"`      // 是否已完成
+	IsOk        bool   `field:"isOk"`        // 是否完全成功
+	IsReady     uint8  `field:"isReady"`     // 是否已准备好
+	Description string `field:"description"` // 描述
+}
+
+type HTTPCacheTaskOperator struct {
+	Id          interface{} // ID
+	UserId      interface{} // 用户ID
+	Type        interface{} // 任务类型：purge|fetch
+	KeyType     interface{} // Key类型
+	State       interface{} // 状态
+	CreatedAt   interface{} // 创建时间
+	DoneAt      interface{} // 完成时间
+	Day         interface{} // 创建日期YYYYMMDD
+	IsDone      interface{} // 是否已完成
+	IsOk        interface{} // 是否完全成功
+	IsReady     interface{} // 是否已准备好
+	Description interface{} // 描述
+}
+
+func NewHTTPCacheTaskOperator() *HTTPCacheTaskOperator {
+	return &HTTPCacheTaskOperator{}
+}

internal/db/models/http_cache_task_model_ext.go

@@ -0,0 +1 @@
+package models

internal/db/models/http_deflate_policy_dao.go

@@ -113,7 +113,7 @@ func (this *HTTPDeflatePolicyDAO) ComposeDeflateConfig(tx *dbs.Tx, policyId int6
 
 // CreatePolicy 创建策略
 func (this *HTTPDeflatePolicyDAO) CreatePolicy(tx *dbs.Tx, level int, minLengthJSON []byte, maxLengthJSON []byte, condsJSON []byte) (int64, error) {
-	op := NewHTTPDeflatePolicyOperator()
+	var op = NewHTTPDeflatePolicyOperator()
 	op.State = HTTPDeflatePolicyStateEnabled
 	op.IsOn = true
 	op.Level = level
@@ -138,7 +138,7 @@ func (this *HTTPDeflatePolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, level
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPDeflatePolicyOperator()
+	var op = NewHTTPDeflatePolicyOperator()
 	op.Id = policyId
 	op.Level = level
 	if len(minLengthJSON) > 0 {

internal/db/models/http_fastcgi_dao.go

@@ -121,7 +121,7 @@ func (this *HTTPFastcgiDAO) ComposeFastcgiConfig(tx *dbs.Tx, fastcgiId int64) (*
 
 // CreateFastcgi 创建Fastcgi
 func (this *HTTPFastcgiDAO) CreateFastcgi(tx *dbs.Tx, adminId int64, userId int64, isOn bool, address string, paramsJSON []byte, readTimeoutJSON []byte, connTimeoutJSON []byte, poolSize int32, pathInfoPattern string) (int64, error) {
-	op := NewHTTPFastcgiOperator()
+	var op = NewHTTPFastcgiOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.IsOn = isOn
@@ -147,7 +147,7 @@ func (this *HTTPFastcgiDAO) UpdateFastcgi(tx *dbs.Tx, fastcgiId int64, isOn bool
 	if fastcgiId <= 0 {
 		return errors.New("invalid 'fastcgiId'")
 	}
-	op := NewHTTPFastcgiOperator()
+	var op = NewHTTPFastcgiOperator()
 	op.Id = fastcgiId
 	op.IsOn = isOn
 	op.Address = address

internal/db/models/http_firewall_policy_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
@@ -116,7 +117,7 @@ func (this *HTTPFirewallPolicyDAO) FindAllEnabledFirewallPolicies(tx *dbs.Tx) (r
 
 // CreateFirewallPolicy 创建策略
 func (this *HTTPFirewallPolicyDAO) CreateFirewallPolicy(tx *dbs.Tx, userId int64, serverGroupId int64, serverId int64, isOn bool, name string, description string, inboundJSON []byte, outboundJSON []byte) (int64, error) {
-	op := NewHTTPFirewallPolicyOperator()
+	var op = NewHTTPFirewallPolicyOperator()
 	op.UserId = userId
 	op.GroupId = serverGroupId
 	op.ServerId = serverId
@@ -130,14 +131,31 @@ func (this *HTTPFirewallPolicyDAO) CreateFirewallPolicy(tx *dbs.Tx, userId int64
 	if len(outboundJSON) > 0 {
 		op.Outbound = outboundJSON
 	}
-	op.UseLocalFirewall = true
 
-	{
-		synFloodJSON, err := json.Marshal(firewallconfigs.DefaultSYNFloodConfig())
+	if userId <= 0 && serverGroupId <=0 && serverId <= 0 {
+		// synFlood
+		var synFloodConfig = firewallconfigs.DefaultSYNFloodConfig()
+		synFloodJSON, err := json.Marshal(synFloodConfig)
 		if err != nil {
 			return 0, err
 		}
 		op.SynFlood = synFloodJSON
+
+		// block options
+		var blockOptions = firewallconfigs.DefaultHTTPFirewallBlockAction()
+		blockOptionsJSON, err := json.Marshal(blockOptions)
+		if err != nil {
+			return 0, err
+		}
+		op.BlockOptions = blockOptionsJSON
+
+		// captcha options
+		var captchaOptions = firewallconfigs.DefaultHTTPFirewallCaptchaAction()
+		captchaOptionsJSON, err := json.Marshal(captchaOptions)
+		if err != nil {
+			return 0, err
+		}
+		op.CaptchaOptions = captchaOptionsJSON
 	}
 
 	err := this.Save(tx, op)
@@ -159,8 +177,8 @@ func (this *HTTPFirewallPolicyDAO) CreateDefaultFirewallPolicy(tx *dbs.Tx, name
 		groupCodes = append(groupCodes, group.Code)
 	}
 
-	inboundConfig := &firewallconfigs.HTTPFirewallInboundConfig{IsOn: true}
-	outboundConfig := &firewallconfigs.HTTPFirewallOutboundConfig{IsOn: true}
+	var inboundConfig = &firewallconfigs.HTTPFirewallInboundConfig{IsOn: true}
+	var outboundConfig = &firewallconfigs.HTTPFirewallOutboundConfig{IsOn: true}
 	if templatePolicy.Inbound != nil {
 		for _, group := range templatePolicy.Inbound.Groups {
 			isOn := lists.ContainsString(groupCodes, group.Code)
@@ -206,6 +224,7 @@ func (this *HTTPFirewallPolicyDAO) CreateDefaultFirewallPolicy(tx *dbs.Tx, name
 	if err != nil {
 		return 0, err
 	}
+
 	return policyId, nil
 }
 
@@ -214,7 +233,7 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInboundAndOutbound(tx *db
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPFirewallPolicyOperator()
+	var op = NewHTTPFirewallPolicyOperator()
 	op.Id = policyId
 	if len(inboundJSON) > 0 {
 		op.Inbound = inboundJSON
@@ -243,7 +262,7 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInbound(tx *dbs.Tx, polic
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPFirewallPolicyOperator()
+	var op = NewHTTPFirewallPolicyOperator()
 	op.Id = policyId
 	if len(inboundJSON) > 0 {
 		op.Inbound = inboundJSON
@@ -259,11 +278,23 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInbound(tx *dbs.Tx, polic
 }
 
 // UpdateFirewallPolicy 修改策略
-func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, inboundJSON []byte, outboundJSON []byte, blockOptionsJSON []byte, mode firewallconfigs.FirewallMode, useLocalFirewall bool, synFloodConfig *firewallconfigs.SYNFloodConfig) error {
+func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx,
+	policyId int64,
+	isOn bool,
+	name string,
+	description string,
+	inboundJSON []byte,
+	outboundJSON []byte,
+	blockOptionsJSON []byte,
+	captchaOptionsJSON []byte,
+	mode firewallconfigs.FirewallMode,
+	useLocalFirewall bool,
+	synFloodConfig *firewallconfigs.SYNFloodConfig,
+	logConfig *firewallconfigs.HTTPFirewallPolicyLogConfig) error {
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPFirewallPolicyOperator()
+	var op = NewHTTPFirewallPolicyOperator()
 	op.Id = policyId
 	op.IsOn = isOn
 	op.Name = name
@@ -279,9 +310,12 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx, policyId int
 	} else {
 		op.Outbound = "null"
 	}
-	if len(blockOptionsJSON) > 0 {
+	if IsNotNull(blockOptionsJSON) {
 		op.BlockOptions = blockOptionsJSON
 	}
+	if IsNotNull(captchaOptionsJSON) {
+		op.CaptchaOptions = captchaOptionsJSON
+	}
 
 	if synFloodConfig != nil {
 		synFloodConfigJSON, err := json.Marshal(synFloodConfig)
@@ -293,6 +327,16 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx, policyId int
 		op.SynFlood = "null"
 	}
 
+	if logConfig != nil {
+		logJSON, err := json.Marshal(logConfig)
+		if err != nil {
+			return err
+		}
+		op.Log = logJSON
+	} else {
+		op.Log = "null"
+	}
+
 	op.UseLocalFirewall = useLocalFirewall
 	err := this.Save(tx, op)
 	if err != nil {
@@ -311,7 +355,7 @@ func (this *HTTPFirewallPolicyDAO) CountAllEnabledFirewallPolicies(tx *dbs.Tx, c
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.
 		State(HTTPFirewallPolicyStateEnabled).
@@ -330,7 +374,7 @@ func (this *HTTPFirewallPolicyDAO) ListEnabledFirewallPolicies(tx *dbs.Tx, clust
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(HTTPFirewallPolicyStateEnabled).
@@ -364,7 +408,7 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 		return nil, nil
 	}
 
-	config := &firewallconfigs.HTTPFirewallPolicy{}
+	var config = &firewallconfigs.HTTPFirewallPolicy{}
 	config.Id = int64(policy.Id)
 	config.IsOn = policy.IsOn
 	config.Name = policy.Name
@@ -434,7 +478,7 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 
 	// Block动作配置
 	if IsNotNull(policy.BlockOptions) {
-		blockAction := &firewallconfigs.HTTPFirewallBlockAction{}
+		var blockAction = &firewallconfigs.HTTPFirewallBlockAction{}
 		err = json.Unmarshal(policy.BlockOptions, blockAction)
 		if err != nil {
 			return config, err
@@ -442,6 +486,16 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 		config.BlockOptions = blockAction
 	}
 
+	// Captcha动作配置
+	if IsNotNull(policy.CaptchaOptions) {
+		var captchaAction = &firewallconfigs.HTTPFirewallCaptchaAction{}
+		err = json.Unmarshal(policy.CaptchaOptions, captchaAction)
+		if err != nil {
+			return config, err
+		}
+		config.CaptchaOptions = captchaAction
+	}
+
 	// syn flood
 	if IsNotNull(policy.SynFlood) {
 		var synFloodConfig = &firewallconfigs.SYNFloodConfig{}
@@ -452,6 +506,18 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 		config.SYNFlood = synFloodConfig
 	}
 
+	// log
+	if IsNotNull(policy.Log) {
+		var logConfig = &firewallconfigs.HTTPFirewallPolicyLogConfig{}
+		err = json.Unmarshal(policy.Log, logConfig)
+		if err != nil {
+			return nil, err
+		}
+		config.Log = logConfig
+	} else {
+		config.Log = firewallconfigs.DefaultHTTPFirewallPolicyLogConfig
+	}
+
 	if cacheMap != nil {
 		cacheMap.Put(cacheKey, config)
 	}
@@ -492,6 +558,7 @@ func (this *HTTPFirewallPolicyDAO) CheckUserFirewallPolicy(tx *dbs.Tx, userId in
 }
 
 // FindEnabledFirewallPolicyIdsWithIPListId 查找包含某个IPList的所有策略
+// TODO 改成通过 serverId 查询
 func (this *HTTPFirewallPolicyDAO) FindEnabledFirewallPolicyIdsWithIPListId(tx *dbs.Tx, ipListId int64) ([]int64, error) {
 	ones, err := this.Query(tx).
 		ResultPk().
@@ -510,6 +577,7 @@ func (this *HTTPFirewallPolicyDAO) FindEnabledFirewallPolicyIdsWithIPListId(tx *
 }
 
 // FindEnabledFirewallPolicyWithIPListId 查找使用某个IPList的策略
+// TODO 改成通过 serverId 查询
 func (this *HTTPFirewallPolicyDAO) FindEnabledFirewallPolicyWithIPListId(tx *dbs.Tx, ipListId int64) (*HTTPFirewallPolicy, error) {
 	one, err := this.Query(tx).
 		State(HTTPFirewallPolicyStateEnabled).

internal/db/models/http_firewall_policy_model.go

@@ -18,9 +18,11 @@ type HTTPFirewallPolicy struct {
 	Inbound          dbs.JSON `field:"inbound"`          // 入站规则
 	Outbound         dbs.JSON `field:"outbound"`         // 出站规则
 	BlockOptions     dbs.JSON `field:"blockOptions"`     // BLOCK选项
+	CaptchaOptions   dbs.JSON `field:"captchaOptions"`   // 验证码选项
 	Mode             string   `field:"mode"`             // 模式
 	UseLocalFirewall uint8    `field:"useLocalFirewall"` // 是否自动使用本地防火墙
 	SynFlood         dbs.JSON `field:"synFlood"`         // SynFlood防御设置
+	Log              dbs.JSON `field:"log"`              // 日志配置
 }
 
 type HTTPFirewallPolicyOperator struct {
@@ -38,9 +40,11 @@ type HTTPFirewallPolicyOperator struct {
 	Inbound          interface{} // 入站规则
 	Outbound         interface{} // 出站规则
 	BlockOptions     interface{} // BLOCK选项
+	CaptchaOptions   interface{} // 验证码选项
 	Mode             interface{} // 模式
 	UseLocalFirewall interface{} // 是否自动使用本地防火墙
 	SynFlood         interface{} // SynFlood防御设置
+	Log              interface{} // 日志配置
 }
 
 func NewHTTPFirewallPolicyOperator() *HTTPFirewallPolicyOperator {

internal/db/models/http_firewall_rule_dao.go

@@ -116,7 +116,7 @@ func (this *HTTPFirewallRuleDAO) ComposeFirewallRule(tx *dbs.Tx, ruleId int64) (
 
 // CreateOrUpdateRuleFromConfig 从配置中配置规则
 func (this *HTTPFirewallRuleDAO) CreateOrUpdateRuleFromConfig(tx *dbs.Tx, ruleConfig *firewallconfigs.HTTPFirewallRule) (int64, error) {
-	op := NewHTTPFirewallRuleOperator()
+	var op = NewHTTPFirewallRuleOperator()
 	op.Id = ruleConfig.Id
 	op.State = HTTPFirewallRuleStateEnabled
 	op.IsOn = ruleConfig.IsOn

internal/db/models/http_firewall_rule_group_dao.go

@@ -120,7 +120,7 @@ func (this *HTTPFirewallRuleGroupDAO) ComposeFirewallRuleGroup(tx *dbs.Tx, group
 
 // CreateGroupFromConfig 从配置中创建分组
 func (this *HTTPFirewallRuleGroupDAO) CreateGroupFromConfig(tx *dbs.Tx, groupConfig *firewallconfigs.HTTPFirewallRuleGroup) (int64, error) {
-	op := NewHTTPFirewallRuleGroupOperator()
+	var op = NewHTTPFirewallRuleGroupOperator()
 	op.IsOn = groupConfig.IsOn
 	op.Name = groupConfig.Name
 	op.Description = groupConfig.Description
@@ -166,7 +166,7 @@ func (this *HTTPFirewallRuleGroupDAO) UpdateGroupIsOn(tx *dbs.Tx, groupId int64,
 
 // CreateGroup 创建分组
 func (this *HTTPFirewallRuleGroupDAO) CreateGroup(tx *dbs.Tx, isOn bool, name string, code string, description string) (int64, error) {
-	op := NewHTTPFirewallRuleGroupOperator()
+	var op = NewHTTPFirewallRuleGroupOperator()
 	op.State = HTTPFirewallRuleStateEnabled
 	op.IsOn = isOn
 	op.Name = name
@@ -184,7 +184,7 @@ func (this *HTTPFirewallRuleGroupDAO) UpdateGroup(tx *dbs.Tx, groupId int64, isO
 	if groupId <= 0 {
 		return errors.New("invalid groupId")
 	}
-	op := NewHTTPFirewallRuleGroupOperator()
+	var op = NewHTTPFirewallRuleGroupOperator()
 	op.Id = groupId
 	op.IsOn = isOn
 	op.Name = name
@@ -202,7 +202,7 @@ func (this *HTTPFirewallRuleGroupDAO) UpdateGroupSets(tx *dbs.Tx, groupId int64,
 	if groupId <= 0 {
 		return errors.New("invalid groupId")
 	}
-	op := NewHTTPFirewallRuleGroupOperator()
+	var op = NewHTTPFirewallRuleGroupOperator()
 	op.Id = groupId
 	op.Sets = setRefsJSON
 	err := this.Save(tx, op)

internal/db/models/http_firewall_rule_set_dao.go

@@ -133,7 +133,7 @@ func (this *HTTPFirewallRuleSetDAO) ComposeFirewallRuleSet(tx *dbs.Tx, setId int
 
 // CreateOrUpdateSetFromConfig 从配置中创建规则集
 func (this *HTTPFirewallRuleSetDAO) CreateOrUpdateSetFromConfig(tx *dbs.Tx, setConfig *firewallconfigs.HTTPFirewallRuleSet) (int64, error) {
-	op := NewHTTPFirewallRuleSetOperator()
+	var op = NewHTTPFirewallRuleSetOperator()
 	op.State = HTTPFirewallRuleSetStateEnabled
 	op.Id = setConfig.Id
 	op.IsOn = setConfig.IsOn

internal/db/models/http_gzip_dao.go

@@ -121,7 +121,7 @@ func (this *HTTPGzipDAO) ComposeGzipConfig(tx *dbs.Tx, gzipId int64) (*servercon
 
 // CreateGzip 创建Gzip
 func (this *HTTPGzipDAO) CreateGzip(tx *dbs.Tx, level int, minLengthJSON []byte, maxLengthJSON []byte, condsJSON []byte) (int64, error) {
-	op := NewHTTPGzipOperator()
+	var op = NewHTTPGzipOperator()
 	op.State = HTTPGzipStateEnabled
 	op.IsOn = true
 	op.Level = level
@@ -146,7 +146,7 @@ func (this *HTTPGzipDAO) UpdateGzip(tx *dbs.Tx, gzipId int64, level int, minLeng
 	if gzipId <= 0 {
 		return errors.New("invalid gzipId")
 	}
-	op := NewHTTPGzipOperator()
+	var op = NewHTTPGzipOperator()
 	op.Id = gzipId
 	op.Level = level
 	if len(minLengthJSON) > 0 {

internal/db/models/http_header_dao.go

@@ -81,7 +81,7 @@ func (this *HTTPHeaderDAO) FindHTTPHeaderName(tx *dbs.Tx, id int64) (string, err
 
 // CreateHeader 创建Header
 func (this *HTTPHeaderDAO) CreateHeader(tx *dbs.Tx, userId int64, name string, value string, status []int, disableRedirect bool, shouldAppend bool, shouldReplace bool, replaceValues []*shared.HTTPHeaderReplaceValue, methods []string, domains []string) (int64, error) {
-	op := NewHTTPHeaderOperator()
+	var op = NewHTTPHeaderOperator()
 	op.UserId = userId
 	op.State = HTTPHeaderStateEnabled
 	op.IsOn = true
@@ -156,7 +156,7 @@ func (this *HTTPHeaderDAO) UpdateHeader(tx *dbs.Tx, headerId int64, name string,
 		return errors.New("invalid headerId")
 	}
 
-	op := NewHTTPHeaderOperator()
+	var op = NewHTTPHeaderOperator()
 	op.Id = headerId
 	op.Name = name
 	op.Value = value

internal/db/models/http_header_policy_dao.go

@@ -77,7 +77,7 @@ func (this *HTTPHeaderPolicyDAO) FindEnabledHTTPHeaderPolicy(tx *dbs.Tx, id int6
 
 // CreateHeaderPolicy 创建策略
 func (this *HTTPHeaderPolicyDAO) CreateHeaderPolicy(tx *dbs.Tx) (int64, error) {
-	op := NewHTTPHeaderPolicyOperator()
+	var op = NewHTTPHeaderPolicyOperator()
 	op.IsOn = true
 	op.State = HTTPHeaderPolicyStateEnabled
 	err := this.Save(tx, op)
@@ -93,7 +93,7 @@ func (this *HTTPHeaderPolicyDAO) UpdateAddingHeaders(tx *dbs.Tx, policyId int64,
 		return errors.New("invalid policyId")
 	}
 
-	op := NewHTTPHeaderPolicyOperator()
+	var op = NewHTTPHeaderPolicyOperator()
 	op.Id = policyId
 	op.AddHeaders = headersJSON
 	err := this.Save(tx, op)
@@ -109,7 +109,7 @@ func (this *HTTPHeaderPolicyDAO) UpdateSettingHeaders(tx *dbs.Tx, policyId int64
 		return errors.New("invalid policyId")
 	}
 
-	op := NewHTTPHeaderPolicyOperator()
+	var op = NewHTTPHeaderPolicyOperator()
 	op.Id = policyId
 	op.SetHeaders = headersJSON
 	err := this.Save(tx, op)
@@ -125,7 +125,7 @@ func (this *HTTPHeaderPolicyDAO) UpdateReplacingHeaders(tx *dbs.Tx, policyId int
 		return errors.New("invalid policyId")
 	}
 
-	op := NewHTTPHeaderPolicyOperator()
+	var op = NewHTTPHeaderPolicyOperator()
 	op.Id = policyId
 	op.ReplaceHeaders = headersJSON
 	err := this.Save(tx, op)
@@ -141,7 +141,7 @@ func (this *HTTPHeaderPolicyDAO) UpdateAddingTrailers(tx *dbs.Tx, policyId int64
 		return errors.New("invalid policyId")
 	}
 
-	op := NewHTTPHeaderPolicyOperator()
+	var op = NewHTTPHeaderPolicyOperator()
 	op.Id = policyId
 	op.AddTrailers = headersJSON
 	err := this.Save(tx, op)
@@ -162,7 +162,7 @@ func (this *HTTPHeaderPolicyDAO) UpdateDeletingHeaders(tx *dbs.Tx, policyId int6
 		return err
 	}
 
-	op := NewHTTPHeaderPolicyOperator()
+	var op = NewHTTPHeaderPolicyOperator()
 	op.Id = policyId
 	op.DeleteHeaders = string(namesJSON)
 	err = this.Save(tx, op)

internal/db/models/http_location_dao.go

@@ -87,7 +87,7 @@ func (this *HTTPLocationDAO) FindHTTPLocationName(tx *dbs.Tx, id int64) (string,
 
 // CreateLocation 创建路由规则
 func (this *HTTPLocationDAO) CreateLocation(tx *dbs.Tx, parentId int64, name string, pattern string, description string, isBreak bool, condsJSON []byte, domains []string) (int64, error) {
-	op := NewHTTPLocationOperator()
+	var op = NewHTTPLocationOperator()
 	op.IsOn = true
 	op.State = HTTPLocationStateEnabled
 	op.ParentId = parentId
@@ -121,7 +121,7 @@ func (this *HTTPLocationDAO) UpdateLocation(tx *dbs.Tx, locationId int64, name s
 	if locationId <= 0 {
 		return errors.New("invalid locationId")
 	}
-	op := NewHTTPLocationOperator()
+	var op = NewHTTPLocationOperator()
 	op.Id = locationId
 	op.Name = name
 	op.Pattern = pattern
@@ -257,7 +257,7 @@ func (this *HTTPLocationDAO) UpdateLocationReverseProxy(tx *dbs.Tx, locationId i
 	if locationId <= 0 {
 		return errors.New("invalid locationId")
 	}
-	op := NewHTTPLocationOperator()
+	var op = NewHTTPLocationOperator()
 	op.Id = locationId
 	op.ReverseProxy = JSONBytes(reverseProxyJSON)
 	err := this.Save(tx, op)
@@ -281,7 +281,7 @@ func (this *HTTPLocationDAO) UpdateLocationWeb(tx *dbs.Tx, locationId int64, web
 	if locationId <= 0 {
 		return errors.New("invalid locationId")
 	}
-	op := NewHTTPLocationOperator()
+	var op = NewHTTPLocationOperator()
 	op.Id = locationId
 	op.WebId = webId
 	err := this.Save(tx, op)

internal/db/models/http_page_dao.go

@@ -77,8 +77,9 @@ func (this *HTTPPageDAO) FindEnabledHTTPPage(tx *dbs.Tx, id int64) (*HTTPPage, e
 }
 
 // CreatePage 创建Page
-func (this *HTTPPageDAO) CreatePage(tx *dbs.Tx, statusList []string, bodyType shared.BodyType, url string, body string, newStatus int) (pageId int64, err error) {
-	op := NewHTTPPageOperator()
+func (this *HTTPPageDAO) CreatePage(tx *dbs.Tx, userId int64, statusList []string, bodyType shared.BodyType, url string, body string, newStatus int) (pageId int64, err error) {
+	var op = NewHTTPPageOperator()
+	op.UserId = userId
 	op.IsOn = true
 	op.State = HTTPPageStateEnabled
 
@@ -107,7 +108,7 @@ func (this *HTTPPageDAO) UpdatePage(tx *dbs.Tx, pageId int64, statusList []strin
 		return errors.New("invalid pageId")
 	}
 
-	op := NewHTTPPageOperator()
+	var op = NewHTTPPageOperator()
 	op.Id = pageId
 	op.IsOn = true
 	op.State = HTTPPageStateEnabled
@@ -182,6 +183,26 @@ func (this *HTTPPageDAO) ComposePageConfig(tx *dbs.Tx, pageId int64, cacheMap *u
 	return config, nil
 }
 
+// CheckUserPage 检查用户页面
+func (this *HTTPPageDAO) CheckUserPage(tx *dbs.Tx, userId int64, pageId int64) error {
+	if userId <= 0 || pageId <= 0 {
+		return ErrNotFound
+	}
+
+	b, err := this.Query(tx).
+		Pk(pageId).
+		Attr("userId", userId).
+		State(HTTPPageStateEnabled).
+		Exist()
+	if err != nil {
+		return err
+	}
+	if !b {
+		return ErrNotFound
+	}
+	return nil
+}
+
 // NotifyUpdate 通知更新
 func (this *HTTPPageDAO) NotifyUpdate(tx *dbs.Tx, pageId int64) error {
 	webId, err := SharedHTTPWebDAO.FindEnabledWebIdWithPageId(tx, pageId)

internal/db/models/http_rewrite_rule_dao.go

@@ -125,7 +125,7 @@ func (this *HTTPRewriteRuleDAO) ComposeRewriteRule(tx *dbs.Tx, rewriteRuleId int
 
 // CreateRewriteRule 创建规则
 func (this *HTTPRewriteRuleDAO) CreateRewriteRule(tx *dbs.Tx, pattern string, replace string, mode string, redirectStatus int, isBreak bool, proxyHost string, withQuery bool, isOn bool, condsJSON []byte) (int64, error) {
-	op := NewHTTPRewriteRuleOperator()
+	var op = NewHTTPRewriteRuleOperator()
 	op.State = HTTPRewriteRuleStateEnabled
 	op.IsOn = isOn
 
@@ -150,7 +150,7 @@ func (this *HTTPRewriteRuleDAO) UpdateRewriteRule(tx *dbs.Tx, rewriteRuleId int6
 	if rewriteRuleId <= 0 {
 		return errors.New("invalid rewriteRuleId")
 	}
-	op := NewHTTPRewriteRuleOperator()
+	var op = NewHTTPRewriteRuleOperator()
 	op.Id = rewriteRuleId
 	op.IsOn = isOn
 	op.Pattern = pattern

internal/db/models/http_web_dao.go

@@ -3,6 +3,7 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
@@ -94,7 +95,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64, cacheMap *util
 		return nil, nil
 	}
 
-	config := &serverconfigs.HTTPWebConfig{}
+	var config = &serverconfigs.HTTPWebConfig{}
 	config.Id = webId
 	config.IsOn = web.IsOn
 
@@ -446,6 +447,16 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64, cacheMap *util
 		}
 	}
 
+	// UAM
+	if teaconst.IsPlus && IsNotNull(web.Uam) {
+		var uamConfig = &serverconfigs.UAMConfig{}
+		err = json.Unmarshal(web.Uam, uamConfig)
+		if err != nil {
+			return nil, err
+		}
+		config.UAM = uamConfig
+	}
+
 	if cacheMap != nil {
 		cacheMap.Put(cacheKey, config)
 	}
@@ -455,7 +466,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64, cacheMap *util
 
 // CreateWeb 创建Web配置
 func (this *HTTPWebDAO) CreateWeb(tx *dbs.Tx, adminId int64, userId int64, rootJSON []byte) (int64, error) {
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.State = HTTPWebStateEnabled
 	op.AdminId = adminId
 	op.UserId = userId
@@ -474,7 +485,7 @@ func (this *HTTPWebDAO) UpdateWeb(tx *dbs.Tx, webId int64, rootJSON []byte) erro
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Root = JSONBytes(rootJSON)
 	err := this.Save(tx, op)
@@ -490,7 +501,7 @@ func (this *HTTPWebDAO) UpdateWebCompression(tx *dbs.Tx, webId int64, compressio
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Compression = JSONBytes(compressionConfig)
 	err := this.Save(tx, op)
@@ -506,7 +517,7 @@ func (this *HTTPWebDAO) UpdateWebWebP(tx *dbs.Tx, webId int64, webpConfig []byte
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Webp = JSONBytes(webpConfig)
 	err := this.Save(tx, op)
@@ -537,7 +548,7 @@ func (this *HTTPWebDAO) UpdateWebCharset(tx *dbs.Tx, webId int64, charsetJSON []
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Charset = JSONBytes(charsetJSON)
 	err := this.Save(tx, op)
@@ -553,7 +564,7 @@ func (this *HTTPWebDAO) UpdateWebRequestHeaderPolicy(tx *dbs.Tx, webId int64, he
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.RequestHeader = JSONBytes(headerPolicyJSON)
 	err := this.Save(tx, op)
@@ -569,7 +580,7 @@ func (this *HTTPWebDAO) UpdateWebResponseHeaderPolicy(tx *dbs.Tx, webId int64, h
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.ResponseHeader = JSONBytes(headerPolicyJSON)
 	err := this.Save(tx, op)
@@ -585,7 +596,7 @@ func (this *HTTPWebDAO) UpdateWebPages(tx *dbs.Tx, webId int64, pagesJSON []byte
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Pages = JSONBytes(pagesJSON)
 	err := this.Save(tx, op)
@@ -601,7 +612,7 @@ func (this *HTTPWebDAO) UpdateWebShutdown(tx *dbs.Tx, webId int64, shutdownJSON
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Shutdown = JSONBytes(shutdownJSON)
 	err := this.Save(tx, op)
@@ -617,7 +628,7 @@ func (this *HTTPWebDAO) UpdateWebAccessLogConfig(tx *dbs.Tx, webId int64, access
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.AccessLog = JSONBytes(accessLogJSON)
 	err := this.Save(tx, op)
@@ -633,7 +644,7 @@ func (this *HTTPWebDAO) UpdateWebStat(tx *dbs.Tx, webId int64, statJSON []byte)
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Stat = JSONBytes(statJSON)
 	err := this.Save(tx, op)
@@ -649,7 +660,7 @@ func (this *HTTPWebDAO) UpdateWebCache(tx *dbs.Tx, webId int64, cacheJSON []byte
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Cache = JSONBytes(cacheJSON)
 	err := this.Save(tx, op)
@@ -665,7 +676,7 @@ func (this *HTTPWebDAO) UpdateWebFirewall(tx *dbs.Tx, webId int64, firewallJSON
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Firewall = JSONBytes(firewallJSON)
 	err := this.Save(tx, op)
@@ -681,7 +692,7 @@ func (this *HTTPWebDAO) UpdateWebLocations(tx *dbs.Tx, webId int64, locationsJSO
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Locations = JSONBytes(locationsJSON)
 	err := this.Save(tx, op)
@@ -697,7 +708,7 @@ func (this *HTTPWebDAO) UpdateWebRedirectToHTTPS(tx *dbs.Tx, webId int64, redire
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.RedirectToHttps = JSONBytes(redirectToHTTPSJSON)
 	err := this.Save(tx, op)
@@ -713,7 +724,7 @@ func (this *HTTPWebDAO) UpdateWebsocket(tx *dbs.Tx, webId int64, websocketJSON [
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Websocket = JSONBytes(websocketJSON)
 	err := this.Save(tx, op)
@@ -729,7 +740,7 @@ func (this *HTTPWebDAO) UpdateWebFastcgi(tx *dbs.Tx, webId int64, fastcgiJSON []
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Fastcgi = JSONBytes(fastcgiJSON)
 	err := this.Save(tx, op)
@@ -745,7 +756,7 @@ func (this *HTTPWebDAO) UpdateWebRewriteRules(tx *dbs.Tx, webId int64, rewriteRu
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.RewriteRules = JSONBytes(rewriteRulesJSON)
 	err := this.Save(tx, op)
@@ -761,7 +772,7 @@ func (this *HTTPWebDAO) UpdateWebAuth(tx *dbs.Tx, webId int64, authJSON []byte)
 	if webId <= 0 {
 		return errors.New("invalid webId")
 	}
-	op := NewHTTPWebOperator()
+	var op = NewHTTPWebOperator()
 	op.Id = webId
 	op.Auth = JSONBytes(authJSON)
 	err := this.Save(tx, op)
@@ -1168,6 +1179,35 @@ func (this *HTTPWebDAO) FindWebRequestScripts(tx *dbs.Tx, webId int64) (*serverc
 	return config, nil
 }
 
+// UpdateWebUAM 开启UAM
+func (this *HTTPWebDAO) UpdateWebUAM(tx *dbs.Tx, webId int64, uamConfig *serverconfigs.UAMConfig) error {
+	if uamConfig == nil {
+		return nil
+	}
+	configJSON, err := json.Marshal(uamConfig)
+	if err != nil {
+		return err
+	}
+
+	err = this.Query(tx).
+		Pk(webId).
+		Set("uam", configJSON).
+		UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	return this.NotifyUpdate(tx, webId)
+}
+
+// FindWebUAM 查找服务的UAM配置
+func (this *HTTPWebDAO) FindWebUAM(tx *dbs.Tx, webId int64) ([]byte, error) {
+	return this.Query(tx).
+		Pk(webId).
+		Result("uam").
+		FindJSONCol()
+}
+
 // NotifyUpdate 通知更新
 func (this *HTTPWebDAO) NotifyUpdate(tx *dbs.Tx, webId int64) error {
 	// server

internal/db/models/http_web_model.go

@@ -37,6 +37,7 @@ type HTTPWeb struct {
 	MergeSlashes       uint8    `field:"mergeSlashes"`       // 是否合并路径中的斜杠
 	RequestLimit       dbs.JSON `field:"requestLimit"`       // 请求限制
 	RequestScripts     dbs.JSON `field:"requestScripts"`     // 请求脚本
+	Uam                dbs.JSON `field:"uam"`                // UAM设置
 }
 
 type HTTPWebOperator struct {
@@ -73,6 +74,7 @@ type HTTPWebOperator struct {
 	MergeSlashes       interface{} // 是否合并路径中的斜杠
 	RequestLimit       interface{} // 请求限制
 	RequestScripts     interface{} // 请求脚本
+	Uam                interface{} // UAM设置
 }
 
 func NewHTTPWebOperator() *HTTPWebOperator {

internal/db/models/http_websocket_dao.go

@@ -110,7 +110,7 @@ func (this *HTTPWebsocketDAO) ComposeWebsocketConfig(tx *dbs.Tx, websocketId int
 
 // CreateWebsocket 创建Websocket配置
 func (this *HTTPWebsocketDAO) CreateWebsocket(tx *dbs.Tx, handshakeTimeoutJSON []byte, allowAllOrigins bool, allowedOrigins []string, requestSameOrigin bool, requestOrigin string) (websocketId int64, err error) {
-	op := NewHTTPWebsocketOperator()
+	var op = NewHTTPWebsocketOperator()
 	op.IsOn = true
 	op.State = HTTPWebsocketStateEnabled
 	if len(handshakeTimeoutJSON) > 0 {
@@ -135,7 +135,7 @@ func (this *HTTPWebsocketDAO) UpdateWebsocket(tx *dbs.Tx, websocketId int64, han
 	if websocketId <= 0 {
 		return errors.New("invalid websocketId")
 	}
-	op := NewHTTPWebsocketOperator()
+	var op = NewHTTPWebsocketOperator()
 	op.Id = websocketId
 	if len(handshakeTimeoutJSON) > 0 {
 		op.HandshakeTimeout = handshakeTimeoutJSON

internal/db/models/ip_item_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -92,6 +93,71 @@ func (this *IPItemDAO) DisableIPItem(tx *dbs.Tx, id int64) error {
 	return this.NotifyUpdate(tx, id)
 }
 
+// DisableIPItemsWithIP 禁用某个IP相关条目
+func (this *IPItemDAO) DisableIPItemsWithIP(tx *dbs.Tx, ipFrom string, ipTo string, userId int64, listId int64) error {
+	if len(ipFrom) == 0 {
+		return errors.New("invalid 'ipFrom'")
+	}
+
+	var query = this.Query(tx).
+		Result("id", "listId").
+		Attr("ipFrom", ipFrom).
+		Attr("ipTo", ipTo).
+		State(IPItemStateEnabled)
+
+	if listId > 0 {
+		if userId > 0 {
+			err := SharedIPListDAO.CheckUserIPList(tx, userId, listId)
+			if err != nil {
+				return err
+			}
+		}
+
+		query.Attr("listId", listId)
+	}
+
+	ones, err := query.FindAll()
+	if err != nil {
+		return err
+	}
+
+	var itemIds = []int64{}
+	for _, one := range ones {
+		var item = one.(*IPItem)
+		var itemId = int64(item.Id)
+		var itemListId = int64(item.ListId)
+		if itemListId != listId && userId > 0 {
+			err = SharedIPListDAO.CheckUserIPList(tx, userId, itemListId)
+			if err != nil {
+				// ignore error
+				continue
+			}
+		}
+		itemIds = append(itemIds, itemId)
+	}
+
+	for _, itemId := range itemIds {
+		version, err := SharedIPListDAO.IncreaseVersion(tx)
+		if err != nil {
+			return err
+		}
+
+		_, err = this.Query(tx).
+			Pk(itemId).
+			Set("state", IPItemStateDisabled).
+			Set("version", version).
+			Update()
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(itemIds) > 0 {
+		return this.NotifyUpdate(tx, itemIds[len(itemIds)-1])
+	}
+	return nil
+}
+
 // DisableIPItemsWithListId 禁用某个IP名单内的所有IP
 func (this *IPItemDAO) DisableIPItemsWithListId(tx *dbs.Tx, listId int64) error {
 	for {
@@ -141,14 +207,35 @@ func (this *IPItemDAO) FindEnabledIPItem(tx *dbs.Tx, id int64) (*IPItem, error)
 
 // DeleteOldItem 根据IP删除以前的旧记录
 func (this *IPItemDAO) DeleteOldItem(tx *dbs.Tx, listId int64, ipFrom string, ipTo string) error {
-	_, err := this.Query(tx).
+	ones, err := this.Query(tx).
+		ResultPk().
 		UseIndex("ipFrom").
 		Attr("listId", listId).
 		Attr("ipFrom", ipFrom).
 		Attr("ipTo", ipTo).
-		Delete()
-	// 这里不通知更新
-	return err
+		Set("state", IPItemStateEnabled).
+		FindAll()
+	if err != nil {
+		return err
+	}
+	for _, one := range ones {
+		var itemId = int64(one.(*IPItem).Id)
+		version, err := SharedIPListDAO.IncreaseVersion(tx)
+		if err != nil {
+			return err
+		}
+
+		err = this.Query(tx).
+			Pk(itemId).
+			Set("version", version).
+			Set("state", IPItemStateDisabled).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 // CreateIPItem 创建IP
@@ -201,6 +288,8 @@ func (this *IPItemDAO) CreateIPItem(tx *dbs.Tx,
 	}
 
 	op.State = IPItemStateEnabled
+	op.UpdatedAt = time.Now().Unix()
+
 	err = this.Save(tx, op)
 	if err != nil {
 		return 0, err
@@ -241,7 +330,7 @@ func (this *IPItemDAO) UpdateIPItem(tx *dbs.Tx, itemId int64, ipFrom string, ipT
 		return err
 	}
 
-	op := NewIPItemOperator()
+	var op = NewIPItemOperator()
 	op.Id = itemId
 	op.IpFrom = ipFrom
 	op.IpTo = ipTo
@@ -264,13 +353,13 @@ func (this *IPItemDAO) UpdateIPItem(tx *dbs.Tx, itemId int64, ipFrom string, ipT
 }
 
 // CountIPItemsWithListId 计算IP数量
-func (this *IPItemDAO) CountIPItemsWithListId(tx *dbs.Tx, listId int64, ipFrom string, ipTo string, keyword string) (int64, error) {
+func (this *IPItemDAO) CountIPItemsWithListId(tx *dbs.Tx, listId int64, ipFrom string, ipTo string, keyword string, eventLevel string) (int64, error) {
 	var query = this.Query(tx).
 		State(IPItemStateEnabled).
 		Attr("listId", listId)
 	if len(keyword) > 0 {
 		query.Where("(ipFrom LIKE :keyword OR ipTo LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(ipFrom) > 0 {
 		query.Attr("ipFrom", ipFrom)
@@ -278,17 +367,20 @@ func (this *IPItemDAO) CountIPItemsWithListId(tx *dbs.Tx, listId int64, ipFrom s
 	if len(ipTo) > 0 {
 		query.Attr("ipTo", ipTo)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
 	return query.Count()
 }
 
 // ListIPItemsWithListId 查找IP列表
-func (this *IPItemDAO) ListIPItemsWithListId(tx *dbs.Tx, listId int64, keyword string, ipFrom string, ipTo string, offset int64, size int64) (result []*IPItem, err error) {
+func (this *IPItemDAO) ListIPItemsWithListId(tx *dbs.Tx, listId int64, keyword string, ipFrom string, ipTo string, eventLevel string, offset int64, size int64) (result []*IPItem, err error) {
 	var query = this.Query(tx).
 		State(IPItemStateEnabled).
 		Attr("listId", listId)
 	if len(keyword) > 0 {
 		query.Where("(ipFrom LIKE :keyword OR ipTo LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(ipFrom) > 0 {
 		query.Attr("ipFrom", ipFrom)
@@ -296,6 +388,9 @@ func (this *IPItemDAO) ListIPItemsWithListId(tx *dbs.Tx, listId int64, keyword s
 	if len(ipTo) > 0 {
 		query.Attr("ipTo", ipTo)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
 	_, err = query.
 		DescPk().
 		Slice(&result).
@@ -371,7 +466,7 @@ func (this *IPItemDAO) ExistsEnabledItem(tx *dbs.Tx, itemId int64) (bool, error)
 }
 
 // CountAllEnabledIPItems 计算数量
-func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool) (int64, error) {
+func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool, eventLevel string, listType string) (int64, error) {
 	var query = this.Query(tx)
 	if len(ip) > 0 {
 		query.Attr("ipFrom", ip)
@@ -379,11 +474,20 @@ func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int6
 	if listId > 0 {
 		query.Attr("listId", listId)
 	} else {
-		query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		if len(listType) > 0 {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1 AND type=:listType))")
+			query.Param("listType", listType)
+		} else {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		}
 	}
 	if unread {
 		query.Attr("isRead", 0)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
+
 	return query.
 		State(IPItemStateEnabled).
 		Where("(expiredAt=0 OR expiredAt>:expiredAt)").
@@ -392,7 +496,7 @@ func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int6
 }
 
 // ListAllEnabledIPItems 搜索所有IP
-func (this *IPItemDAO) ListAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool, offset int64, size int64) (result []*IPItem, err error) {
+func (this *IPItemDAO) ListAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool, eventLevel string, listType string, offset int64, size int64) (result []*IPItem, err error) {
 	var query = this.Query(tx)
 	if len(ip) > 0 {
 		query.Attr("ipFrom", ip)
@@ -400,11 +504,19 @@ func (this *IPItemDAO) ListAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64
 	if listId > 0 {
 		query.Attr("listId", listId)
 	} else {
-		query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		if len(listType) > 0 {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1 AND type=:listType))")
+			query.Param("listType", listType)
+		} else {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		}
 	}
 	if unread {
 		query.Attr("isRead", 0)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
 	_, err = query.
 		State(IPItemStateEnabled).
 		Where("(expiredAt=0 OR expiredAt>:expiredAt)").
@@ -429,7 +541,7 @@ func (this *IPItemDAO) UpdateItemsRead(tx *dbs.Tx) error {
 func (this *IPItemDAO) CleanExpiredIPItems(tx *dbs.Tx) error {
 	// 删除 N 天之前过期的数据
 	_, err := this.Query(tx).
-		Where("expiredAt<=:timestamp").
+		Where("(createdAt<=:timestamp AND updatedAt<=:timestamp)").
 		State(IPItemStateDisabled).
 		Param("timestamp", time.Now().Unix()-7*86400). // N 天之前过期的
 		Limit(10000).                                  // 限制条数，防止数量过多导致超时

internal/db/models/ip_item_dao_test.go

@@ -1,6 +1,7 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/dbs"
@@ -14,7 +15,7 @@ func TestIPItemDAO_NotifyClustersUpdate(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	err := SharedIPItemDAO.NotifyUpdate(tx, 28)
+	err := models.SharedIPItemDAO.NotifyUpdate(tx, 28)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -25,7 +26,7 @@ func TestIPItemDAO_DisableIPItemsWithListId(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	err := SharedIPItemDAO.DisableIPItemsWithListId(tx, 67)
+	err := models.SharedIPItemDAO.DisableIPItemsWithListId(tx, 67)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -36,7 +37,7 @@ func TestIPItemDAO_ListIPItemsAfterVersion(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	_, err := SharedIPItemDAO.ListIPItemsAfterVersion(tx, 0, 100)
+	_, err := models.SharedIPItemDAO.ListIPItemsAfterVersion(tx, 0, 100)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -47,10 +48,10 @@ func TestIPItemDAO_CreateManyIPs(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	var dao = NewIPItemDAO()
+	var dao = models.NewIPItemDAO()
 	var n = 10
 	for i := 0; i < n; i++ {
-		itemId, err := dao.CreateIPItem(tx, firewallconfigs.GlobalListId, "192."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255)), "", time.Now().Unix()+86400, "test", IPItemTypeIPv4, "warning", 0, 0, 0, 0, 0, 0, 0)
+		itemId, err := dao.CreateIPItem(tx, firewallconfigs.GlobalListId, "192."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255)), "", time.Now().Unix()+86400, "test", models.IPItemTypeIPv4, "warning", 0, 0, 0, 0, 0, 0, 0)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -62,3 +63,14 @@ func TestIPItemDAO_CreateManyIPs(t *testing.T) {
 	}
 	t.Log("ok")
 }
+
+func TestIPItemDAO_DisableIPItemsWithIP(t *testing.T) {
+	dbs.NotifyReady()
+
+	var tx *dbs.Tx
+	err := models.SharedIPItemDAO.DisableIPItemsWithIP(tx, "192.168.1.100", "", 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

internal/db/models/ip_library_dao.go

@@ -92,7 +92,7 @@ func (this *IPLibraryDAO) FindLatestIPLibraryWithType(tx *dbs.Tx, libraryType st
 
 // 创建新的IP库
 func (this *IPLibraryDAO) CreateIPLibrary(tx *dbs.Tx, libraryType string, fileId int64) (int64, error) {
-	op := NewIPLibraryOperator()
+	var op = NewIPLibraryOperator()
 	op.Type = libraryType
 	op.FileId = fileId
 	op.State = IPLibraryStateEnabled

internal/db/models/ip_list_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
@@ -137,10 +138,11 @@ func (this *IPListDAO) FindIPListCacheable(tx *dbs.Tx, listId int64) (*IPList, e
 }
 
 // CreateIPList 创建名单
-func (this *IPListDAO) CreateIPList(tx *dbs.Tx, userId int64, listType ipconfigs.IPListType, name string, code string, timeoutJSON []byte, description string, isPublic bool, isGlobal bool) (int64, error) {
-	op := NewIPListOperator()
+func (this *IPListDAO) CreateIPList(tx *dbs.Tx, userId int64, serverId int64, listType ipconfigs.IPListType, name string, code string, timeoutJSON []byte, description string, isPublic bool, isGlobal bool) (int64, error) {
+	var op = NewIPListOperator()
 	op.IsOn = true
 	op.UserId = userId
+	op.ServerId = serverId
 	op.State = IPListStateEnabled
 	op.Type = listType
 	op.Name = name
@@ -163,7 +165,7 @@ func (this *IPListDAO) UpdateIPList(tx *dbs.Tx, listId int64, name string, code
 	if listId <= 0 {
 		return errors.New("invalid listId")
 	}
-	op := NewIPListOperator()
+	var op = NewIPListOperator()
 	op.Id = listId
 	op.Name = name
 	op.Code = code
@@ -188,26 +190,25 @@ func (this *IPListDAO) CheckUserIPList(tx *dbs.Tx, userId int64, listId int64) e
 		return ErrNotFound
 	}
 
-	ok, err := this.Query(tx).
+	// 获取名单信息
+	listOne, err := this.Query(tx).
 		Pk(listId).
-		Attr("userId", userId).
-		Exist()
+		Result("userId", "serverId").
+		Find()
 	if err != nil {
 		return err
 	}
-	if ok {
+	if listOne == nil {
+		return ErrNotFound
+	}
+	var list = listOne.(*IPList)
+	if int64(list.UserId) == userId {
 		return nil
 	}
 
-	// 检查是否被用户的服务所使用
-	policyIds, err := SharedHTTPFirewallPolicyDAO.FindEnabledFirewallPolicyIdsWithIPListId(tx, listId)
-	if err != nil {
-		return err
-	}
-	for _, policyId := range policyIds {
-		if SharedHTTPFirewallPolicyDAO.CheckUserFirewallPolicy(tx, userId, policyId) == nil {
-			return nil
-		}
+	var serverId = int64(list.ServerId)
+	if serverId > 0 {
+		return SharedServerDAO.CheckUserServer(tx, userId, serverId)
 	}
 
 	return ErrNotFound
@@ -221,7 +222,7 @@ func (this *IPListDAO) CountAllEnabledIPLists(tx *dbs.Tx, listType string, isPub
 		Attr("isPublic", isPublic)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.Count()
 }
@@ -234,7 +235,7 @@ func (this *IPListDAO) ListEnabledIPLists(tx *dbs.Tx, listType string, isPublic
 		Attr("isPublic", isPublic)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.Offset(offset).
 		Limit(size).

internal/db/models/ip_list_dao_test.go

@@ -20,6 +20,39 @@ func TestIPListDAO_IncreaseVersion(t *testing.T) {
 	t.Log("version:", version)
 }
 
+func TestIPListDAO_CheckUserIPList(t *testing.T) {
+	dbs.NotifyReady()
+
+	var tx *dbs.Tx
+
+	{
+		err := NewIPListDAO().CheckUserIPList(tx, 1, 100)
+		if err == ErrNotFound {
+			t.Log("not found")
+		} else {
+			t.Log(err)
+		}
+	}
+
+	{
+		err := NewIPListDAO().CheckUserIPList(tx, 1, 85)
+		if err == ErrNotFound {
+			t.Log("not found")
+		} else {
+			t.Log(err)
+		}
+	}
+
+	{
+		err := NewIPListDAO().CheckUserIPList(tx, 1, 17)
+		if err == ErrNotFound {
+			t.Log("not found")
+		} else {
+			t.Log(err)
+		}
+	}
+}
+
 func BenchmarkIPListDAO_IncreaseVersion(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
@@ -32,3 +65,4 @@ func BenchmarkIPListDAO_IncreaseVersion(b *testing.B) {
 		_, _ = dao.IncreaseVersion(tx)
 	}
 }
+

internal/db/models/ip_list_model.go

@@ -9,6 +9,7 @@ type IPList struct {
 	Type        string   `field:"type"`        // 类型
 	AdminId     uint32   `field:"adminId"`     // 用户ID
 	UserId      uint32   `field:"userId"`      // 用户ID
+	ServerId    uint64   `field:"serverId"`    // 服务ID
 	Name        string   `field:"name"`        // 列表名
 	Code        string   `field:"code"`        // 代号
 	State       uint8    `field:"state"`       // 状态
@@ -26,6 +27,7 @@ type IPListOperator struct {
 	Type        interface{} // 类型
 	AdminId     interface{} // 用户ID
 	UserId      interface{} // 用户ID
+	ServerId    interface{} // 服务ID
 	Name        interface{} // 列表名
 	Code        interface{} // 代号
 	State       interface{} // 状态

internal/db/models/log_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
@@ -35,7 +36,7 @@ func init() {
 
 // CreateLog 创建管理员日志
 func (this *LogDAO) CreateLog(tx *dbs.Tx, adminType string, adminId int64, level string, description string, action string, ip string) error {
-	op := NewLogOperator()
+	var op = NewLogOperator()
 	op.Level = level
 	op.Description = description
 	op.Action = action
@@ -72,7 +73,7 @@ func (this *LogDAO) CountLogs(tx *dbs.Tx, dayFrom string, dayTo string, keyword
 	}
 	if len(keyword) > 0 {
 		query.Where("(description LIKE :keyword OR ip LIKE :keyword OR action LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	// 用户类型
@@ -100,7 +101,7 @@ func (this *LogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64, dayFrom strin
 	}
 	if len(keyword) > 0 {
 		query.Where("(description LIKE :keyword OR ip LIKE :keyword OR action LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	// 用户类型

internal/db/models/login_dao.go

@@ -40,7 +40,7 @@ func init() {
 	})
 }
 
-// 启用条目
+// EnableLogin 启用条目
 func (this *LoginDAO) EnableLogin(tx *dbs.Tx, id int64) error {
 	_, err := this.Query(tx).
 		Pk(id).
@@ -49,7 +49,7 @@ func (this *LoginDAO) EnableLogin(tx *dbs.Tx, id int64) error {
 	return err
 }
 
-// 禁用条目
+// DisableLogin 禁用条目
 func (this *LoginDAO) DisableLogin(tx *dbs.Tx, id int64) error {
 	_, err := this.Query(tx).
 		Pk(id).
@@ -58,7 +58,7 @@ func (this *LoginDAO) DisableLogin(tx *dbs.Tx, id int64) error {
 	return err
 }
 
-// 查找启用中的条目
+// FindEnabledLogin 查找启用中的条目
 func (this *LoginDAO) FindEnabledLogin(tx *dbs.Tx, id int64) (*Login, error) {
 	result, err := this.Query(tx).
 		Pk(id).
@@ -70,7 +70,7 @@ func (this *LoginDAO) FindEnabledLogin(tx *dbs.Tx, id int64) (*Login, error) {
 	return result.(*Login), err
 }
 
-// 创建认证
+// CreateLogin 创建认证
 func (this *LoginDAO) CreateLogin(tx *dbs.Tx, Id int64, loginType LoginType, params maps.Map) (int64, error) {
 	if Id <= 0 {
 		return 0, errors.New("invalid Id")
@@ -78,7 +78,7 @@ func (this *LoginDAO) CreateLogin(tx *dbs.Tx, Id int64, loginType LoginType, par
 	if params == nil {
 		params = maps.Map{}
 	}
-	op := NewLoginOperator()
+	var op = NewLoginOperator()
 	op.Id = Id
 	op.Type = loginType
 	op.Params = params.AsJSON()
@@ -87,23 +87,34 @@ func (this *LoginDAO) CreateLogin(tx *dbs.Tx, Id int64, loginType LoginType, par
 	return this.SaveInt64(tx, op)
 }
 
-// 修改认证
-func (this *LoginDAO) UpdateLogin(tx *dbs.Tx, adminId int64, loginType LoginType, params maps.Map, isOn bool) error {
+// UpdateLogin 修改认证
+func (this *LoginDAO) UpdateLogin(tx *dbs.Tx, adminId int64, userId int64, loginType LoginType, params maps.Map, isOn bool) error {
+	if adminId <= 0 && userId <= 0 {
+		return errors.New("invalid adminId and userId")
+	}
+
 	// 是否已经存在
-	loginId, err := this.Query(tx).
-		Attr("adminId", adminId).
+	var query = this.Query(tx).
 		Attr("type", loginType).
 		State(LoginStateEnabled).
-		ResultPk().
-		FindInt64Col(0)
+		ResultPk()
+
+	if adminId > 0 {
+		query.Attr("adminId", adminId)
+	} else if userId > 0 {
+		query.Attr("userId", userId)
+	}
+
+	loginId, err := query.FindInt64Col(0)
 	if err != nil {
 		return err
 	}
-	op := NewLoginOperator()
+	var op = NewLoginOperator()
 	if loginId > 0 {
 		op.Id = loginId
 	} else {
 		op.AdminId = adminId
+		op.UserId = userId
 		op.Type = loginType
 		op.State = LoginStateEnabled
 	}
@@ -117,35 +128,54 @@ func (this *LoginDAO) UpdateLogin(tx *dbs.Tx, adminId int64, loginType LoginType
 	return this.Save(tx, op)
 }
 
-// 禁用相关认证
-func (this *LoginDAO) DisableLoginWithAdminId(tx *dbs.Tx, adminId int64, loginType LoginType) error {
-	_, err := this.Query(tx).
-		Attr("adminId", adminId).
+// DisableLoginWithType 禁用相关认证
+func (this *LoginDAO) DisableLoginWithType(tx *dbs.Tx, adminId int64, userId int64, loginType LoginType) error {
+	var query = this.Query(tx).
 		Attr("type", loginType).
-		Set("isOn", false).
+		Set("isOn", false)
+
+	if adminId > 0 {
+		query.Attr("adminId", adminId)
+	} else if userId > 0 {
+		query.Attr("userId", userId)
+	}
+
+	_, err := query.
 		Update()
 	return err
 }
 
-// 查找管理员相关的认证
-func (this *LoginDAO) FindEnabledLoginWithAdminId(tx *dbs.Tx, adminId int64, loginType LoginType) (*Login, error) {
-	one, err := this.Query(tx).
-		Attr("adminId", adminId).
+// FindEnabledLoginWithType 查找管理员和用户相关的认证
+func (this *LoginDAO) FindEnabledLoginWithType(tx *dbs.Tx, adminId int64, userId int64, loginType LoginType) (*Login, error) {
+	var query = this.Query(tx).
 		Attr("type", loginType).
-		State(LoginStateEnabled).
-		Find()
+		State(LoginStateEnabled)
+
+	if adminId > 0 {
+		query.Attr("adminId", adminId)
+	} else if userId > 0 {
+		query.Attr("userId", userId)
+	}
+
+	one, err := query.Find()
 	if err != nil || one == nil {
 		return nil, err
 	}
 	return one.(*Login), nil
 }
 
-// 检查某个认证是否启用
-func (this *LoginDAO) CheckLoginIsOn(tx *dbs.Tx, adminId int64, loginType LoginType) (bool, error) {
-	return this.Query(tx).
-		Attr("adminId", adminId).
+// CheckLoginIsOn 检查某个认证是否启用
+func (this *LoginDAO) CheckLoginIsOn(tx *dbs.Tx, adminId int64, userId int64, loginType LoginType) (bool, error) {
+	var query = this.Query(tx).
 		Attr("type", loginType).
 		State(LoginStateEnabled).
-		Attr("isOn", true).
-		Exist()
+		Attr("isOn", true)
+
+	if adminId > 0 {
+		query.Attr("adminId", adminId)
+	} else if userId > 0 {
+		query.Attr("userId", userId)
+	}
+
+	return query.Exist()
 }

internal/db/models/message_dao.go

@@ -154,7 +154,7 @@ func (this *MessageDAO) CreateNodeMessage(tx *dbs.Tx, role string, clusterId int
 
 // CreateMessage 创建普通消息
 func (this *MessageDAO) CreateMessage(tx *dbs.Tx, adminId int64, userId int64, messageType MessageType, level string, subject string, body string, paramsJSON []byte) error {
-	op := NewMessageOperator()
+	var op = NewMessageOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.Type = messageType
@@ -230,7 +230,7 @@ func (this *MessageDAO) UpdateMessageRead(tx *dbs.Tx, messageId int64, b bool) e
 	if messageId <= 0 {
 		return errors.New("invalid messageId")
 	}
-	op := NewMessageOperator()
+	var op = NewMessageOperator()
 	op.Id = messageId
 	op.IsRead = b
 	err := this.Save(tx, op)
@@ -286,7 +286,7 @@ func (this *MessageDAO) createMessage(tx *dbs.Tx, role string, clusterId int64,
 	// TODO 检查同样的消息最近是否发送过
 
 	// 创建新消息
-	op := NewMessageOperator()
+	var op = NewMessageOperator()
 	op.AdminId = 0 // TODO
 	op.UserId = 0  // TODO
 	op.Role = role

internal/db/models/message_media_dao.go

@@ -104,7 +104,7 @@ func (this *MessageMediaDAO) UpdateMessageMedias(tx *dbs.Tx, mediaMaps []maps.Ma
 		if err != nil {
 			return err
 		}
-		op := NewMessageMediaOperator()
+		var op = NewMessageMediaOperator()
 		if mediaId > 0 {
 			op.Id = mediaId
 		}

internal/db/models/message_media_instance_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	_ "github.com/go-sql-driver/mysql"
@@ -83,7 +84,7 @@ func (this *MessageMediaInstanceDAO) FindEnabledMessageMediaInstance(tx *dbs.Tx,
 
 // CreateMediaInstance 创建媒介实例
 func (this *MessageMediaInstanceDAO) CreateMediaInstance(tx *dbs.Tx, name string, mediaType string, params maps.Map, description string, rateJSON []byte, hashLifeSeconds int32) (int64, error) {
-	op := NewMessageMediaInstanceOperator()
+	var op = NewMessageMediaInstanceOperator()
 	op.Name = name
 	op.MediaType = mediaType
 
@@ -115,7 +116,7 @@ func (this *MessageMediaInstanceDAO) UpdateMediaInstance(tx *dbs.Tx, instanceId
 		return errors.New("invalid instanceId")
 	}
 
-	op := NewMessageMediaInstanceOperator()
+	var op = NewMessageMediaInstanceOperator()
 	op.Id = instanceId
 	op.Name = name
 	op.MediaType = mediaType
@@ -149,7 +150,7 @@ func (this *MessageMediaInstanceDAO) CountAllEnabledMediaInstances(tx *dbs.Tx, m
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.
 		State(MessageMediaInstanceStateEnabled).
@@ -165,7 +166,7 @@ func (this *MessageMediaInstanceDAO) ListAllEnabledMediaInstances(tx *dbs.Tx, me
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(MessageMediaInstanceStateEnabled).

internal/db/models/message_receiver_dao.go

@@ -76,7 +76,7 @@ func (this *MessageReceiverDAO) DisableReceivers(tx *dbs.Tx, clusterId int64, no
 
 // CreateReceiver 创建接收人
 func (this *MessageReceiverDAO) CreateReceiver(tx *dbs.Tx, role string, clusterId int64, nodeId int64, serverId int64, messageType MessageType, params maps.Map, recipientId int64, recipientGroupId int64) (int64, error) {
-	op := NewMessageReceiverOperator()
+	var op = NewMessageReceiverOperator()
 	op.Role = role
 	op.ClusterId = clusterId
 	op.NodeId = nodeId

internal/db/models/message_recipient_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
@@ -85,7 +86,7 @@ func (this *MessageRecipientDAO) FindEnabledMessageRecipient(tx *dbs.Tx, recipie
 
 // CreateRecipient 创建接收人
 func (this *MessageRecipientDAO) CreateRecipient(tx *dbs.Tx, adminId int64, instanceId int64, user string, groupIds []int64, description string, timeFrom string, timeTo string) (int64, error) {
-	op := NewMessageRecipientOperator()
+	var op = NewMessageRecipientOperator()
 	op.AdminId = adminId
 	op.InstanceId = instanceId
 	op.User = user
@@ -121,7 +122,7 @@ func (this *MessageRecipientDAO) UpdateRecipient(tx *dbs.Tx, recipientId int64,
 		return errors.New("invalid recipientId")
 	}
 
-	op := NewMessageRecipientOperator()
+	var op = NewMessageRecipientOperator()
 	op.Id = recipientId
 	op.AdminId = adminId
 	op.InstanceId = instanceId
@@ -172,7 +173,7 @@ func (this *MessageRecipientDAO) CountAllEnabledRecipients(tx *dbs.Tx, adminId i
 	}
 	if len(keyword) > 0 {
 		query.Where("(`user` LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.
 		State(MessageRecipientStateEnabled).
@@ -197,7 +198,7 @@ func (this *MessageRecipientDAO) ListAllEnabledRecipients(tx *dbs.Tx, adminId in
 	}
 	if len(keyword) > 0 {
 		query.Where("(`user` LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(MessageRecipientStateEnabled).

internal/db/models/message_recipient_group_dao.go

@@ -73,7 +73,7 @@ func (this *MessageRecipientGroupDAO) FindMessageRecipientGroupName(tx *dbs.Tx,
 
 // 创建分组
 func (this *MessageRecipientGroupDAO) CreateGroup(tx *dbs.Tx, name string) (int64, error) {
-	op := NewMessageRecipientGroupOperator()
+	var op = NewMessageRecipientGroupOperator()
 	op.Name = name
 	op.IsOn = true
 	op.State = MessageRecipientStateEnabled
@@ -85,7 +85,7 @@ func (this *MessageRecipientGroupDAO) UpdateGroup(tx *dbs.Tx, groupId int64, nam
 	if groupId <= 0 {
 		return errors.New("invalid groupId")
 	}
-	op := NewMessageRecipientGroupOperator()
+	var op = NewMessageRecipientGroupOperator()
 	op.Id = groupId
 	op.Name = name
 	op.IsOn = isOn

internal/db/models/message_task_dao.go

@@ -131,7 +131,7 @@ func (this *MessageTaskDAO) CreateMessageTask(tx *dbs.Tx, recipientId int64, ins
 		}
 	}
 
-	op := NewMessageTaskOperator()
+	var op = NewMessageTaskOperator()
 	op.RecipientId = recipientId
 	op.InstanceId = instanceId
 	op.Hash = hash
@@ -190,7 +190,7 @@ func (this *MessageTaskDAO) UpdateMessageTaskStatus(tx *dbs.Tx, taskId int64, st
 	if taskId <= 0 {
 		return errors.New("invalid taskId")
 	}
-	op := NewMessageTaskOperator()
+	var op = NewMessageTaskOperator()
 	op.Id = taskId
 	op.Status = status
 	op.SentAt = time.Now().Unix()

internal/db/models/message_task_log_dao.go

@@ -49,7 +49,7 @@ func init() {
 
 // CreateLog 创建日志
 func (this *MessageTaskLogDAO) CreateLog(tx *dbs.Tx, taskId int64, isOk bool, errMsg string, response string) error {
-	op := NewMessageTaskLogOperator()
+	var op = NewMessageTaskLogOperator()
 	op.TaskId = taskId
 	op.IsOk = isOk
 	op.Error = errMsg

internal/db/models/metric_chart_dao.go

@@ -75,7 +75,7 @@ func (this *MetricChartDAO) FindMetricChartName(tx *dbs.Tx, chartId int64) (stri
 
 // CreateChart 创建图表
 func (this *MetricChartDAO) CreateChart(tx *dbs.Tx, itemId int64, name string, chartType string, widthDiv int32, maxItems int32, params maps.Map, ignoreEmptyKeys bool, ignoredKeys []string) (int64, error) {
-	op := NewMetricChartOperator()
+	var op = NewMetricChartOperator()
 	op.ItemId = itemId
 	op.Name = name
 	op.Type = chartType
@@ -112,7 +112,7 @@ func (this *MetricChartDAO) UpdateChart(tx *dbs.Tx, chartId int64, name string,
 	if chartId <= 0 {
 		return errors.New("invalid chartId")
 	}
-	op := NewMetricChartOperator()
+	var op = NewMetricChartOperator()
 	op.Id = chartId
 	op.Name = name
 	op.Type = chartType

internal/db/models/metric_item_dao.go

@@ -12,6 +12,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	"sort"
 	"strings"
+	"sync"
 )
 
 const (
@@ -34,6 +35,9 @@ func NewMetricItemDAO() *MetricItemDAO {
 
 var SharedMetricItemDAO *MetricItemDAO
 
+var metricItemLastTimeCacheMap = map[int64]string{} // itemId => time
+var metricItemLastTimeCacheLocker = &sync.Mutex{}
+
 func init() {
 	dbs.OnReady(func() {
 		SharedMetricItemDAO = NewMetricItemDAO()
@@ -108,10 +112,10 @@ func (this *MetricItemDAO) FindMetricItemName(tx *dbs.Tx, id int64) (string, err
 }
 
 // CreateItem 创建指标
-func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string, name string, keys []string, period int32, periodUnit string, value string, isPublic bool) (int64, error) {
+func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string, name string, keys []string, period int32, periodUnit string, expiresPeriod int32, value string, isPublic bool) (int64, error) {
 	sort.Strings(keys)
 
-	op := NewMetricItemOperator()
+	var op = NewMetricItemOperator()
 	op.Code = code
 	op.Category = category
 	op.Name = name
@@ -126,6 +130,7 @@ func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string,
 	}
 	op.Period = period
 	op.PeriodUnit = periodUnit
+	op.ExpiresPeriod = expiresPeriod
 	op.Value = value
 	op.IsPublic = isPublic
 	op.IsOn = true
@@ -146,7 +151,7 @@ func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string,
 }
 
 // UpdateItem 修改\指标
-func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, keys []string, period int32, periodUnit string, value string, isOn bool, isPublic bool) error {
+func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, keys []string, period int32, periodUnit string, expiresPeriod int32, value string, isOn bool, isPublic bool) error {
 	if itemId <= 0 {
 		return errors.New("invalid itemId")
 	}
@@ -168,7 +173,7 @@ func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, key
 	}
 
 	// 保存
-	op := NewMetricItemOperator()
+	var op = NewMetricItemOperator()
 	op.Id = itemId
 	op.Name = name
 	if len(keys) > 0 {
@@ -182,6 +187,7 @@ func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, key
 	}
 	op.Period = period
 	op.PeriodUnit = periodUnit
+	op.ExpiresPeriod = expiresPeriod
 	op.Value = value
 	op.IsOn = isOn
 	if versionChanged {
@@ -282,14 +288,15 @@ func (this *MetricItemDAO) ComposeItemConfig(tx *dbs.Tx, itemId int64) (*serverc
 	}
 	var item = one.(*MetricItem)
 	var config = &serverconfigs.MetricItemConfig{
-		Id:         int64(item.Id),
-		IsOn:       item.IsOn,
-		Period:     types.Int(item.Period),
-		PeriodUnit: item.PeriodUnit,
-		Category:   item.Category,
-		Value:      item.Value,
-		Keys:       item.DecodeKeys(),
-		Version:    types.Int32(item.Version),
+		Id:            int64(item.Id),
+		IsOn:          item.IsOn,
+		Period:        types.Int(item.Period),
+		PeriodUnit:    item.PeriodUnit,
+		ExpiresPeriod: types.Int(item.ExpiresPeriod),
+		Category:      item.Category,
+		Value:         item.Value,
+		Keys:          item.DecodeKeys(),
+		Version:       types.Int32(item.Version),
 	}
 
 	return config, nil
@@ -301,14 +308,15 @@ func (this *MetricItemDAO) ComposeItemConfigWithItem(item *MetricItem) *serverco
 		return nil
 	}
 	var config = &serverconfigs.MetricItemConfig{
-		Id:         int64(item.Id),
-		IsOn:       item.IsOn,
-		Period:     types.Int(item.Period),
-		PeriodUnit: item.PeriodUnit,
-		Category:   item.Category,
-		Value:      item.Value,
-		Keys:       item.DecodeKeys(),
-		Version:    types.Int32(item.Version),
+		Id:            int64(item.Id),
+		IsOn:          item.IsOn,
+		Period:        types.Int(item.Period),
+		PeriodUnit:    item.PeriodUnit,
+		ExpiresPeriod: types.Int(item.ExpiresPeriod),
+		Category:      item.Category,
+		Value:         item.Value,
+		Keys:          item.DecodeKeys(),
+		Version:       types.Int32(item.Version),
 	}
 
 	return config
@@ -326,6 +334,32 @@ func (this *MetricItemDAO) FindItemVersion(tx *dbs.Tx, itemId int64) (int32, err
 	return types.Int32(version), nil
 }
 
+// UpdateMetricLastTime 更新指标最新数据的时间
+func (this *MetricItemDAO) UpdateMetricLastTime(tx *dbs.Tx, itemId int64, lastTime string) error {
+	metricItemLastTimeCacheLocker.Lock()
+	cachedTime, ok := metricItemLastTimeCacheMap[itemId]
+	if ok && cachedTime == lastTime {
+		metricItemLastTimeCacheLocker.Unlock()
+		return nil
+	}
+
+	metricItemLastTimeCacheMap[itemId] = lastTime
+	metricItemLastTimeCacheLocker.Unlock()
+
+	return this.Query(tx).
+		Pk(itemId).
+		Set("lastTime", lastTime).
+		UpdateQuickly()
+}
+
+// FindMetricLastTime 读取指标最新数据的时间
+func (this *MetricItemDAO) FindMetricLastTime(tx *dbs.Tx, itemId int64) (string, error) {
+	return this.Query(tx).
+		Result("lastTime").
+		Pk(itemId).
+		FindStringCol("")
+}
+
 // NotifyUpdate 通知更新
 func (this *MetricItemDAO) NotifyUpdate(tx *dbs.Tx, itemId int64, isPublic bool) error {
 	if isPublic {

internal/db/models/metric_item_dao_test.go

@@ -1,6 +1,13 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
+	"testing"
 )
+
+func TestMetricStatDAO_Clean(t *testing.T) {
+	var dao = models.NewMetricStatDAO()
+	t.Log(dao.Clean(nil))
+}

internal/db/models/metric_item_model.go

@@ -4,37 +4,41 @@ import "github.com/iwind/TeaGo/dbs"
 
 // MetricItem 指标定义
 type MetricItem struct {
-	Id         uint64   `field:"id"`         // ID
-	IsOn       bool     `field:"isOn"`       // 是否启用
-	Code       string   `field:"code"`       // 代号（用来区分是否内置）
-	Category   string   `field:"category"`   // 类型，比如http, tcp等
-	AdminId    uint32   `field:"adminId"`    // 管理员ID
-	UserId     uint32   `field:"userId"`     // 用户ID
-	Name       string   `field:"name"`       // 指标名称
-	Keys       dbs.JSON `field:"keys"`       // 统计的Key
-	Period     uint32   `field:"period"`     // 周期
-	PeriodUnit string   `field:"periodUnit"` // 周期单位
-	Value      string   `field:"value"`      // 值运算
-	State      uint8    `field:"state"`      // 状态
-	Version    uint32   `field:"version"`    // 版本号
-	IsPublic   bool     `field:"isPublic"`   // 是否为公用
+	Id            uint64   `field:"id"`            // ID
+	IsOn          bool     `field:"isOn"`          // 是否启用
+	Code          string   `field:"code"`          // 代号（用来区分是否内置）
+	Category      string   `field:"category"`      // 类型，比如http, tcp等
+	AdminId       uint32   `field:"adminId"`       // 管理员ID
+	UserId        uint32   `field:"userId"`        // 用户ID
+	Name          string   `field:"name"`          // 指标名称
+	Keys          dbs.JSON `field:"keys"`          // 统计的Key
+	Period        uint32   `field:"period"`        // 周期
+	PeriodUnit    string   `field:"periodUnit"`    // 周期单位
+	ExpiresPeriod uint32   `field:"expiresPeriod"` // 过期周期
+	Value         string   `field:"value"`         // 值运算
+	State         uint8    `field:"state"`         // 状态
+	Version       uint32   `field:"version"`       // 版本号
+	IsPublic      bool     `field:"isPublic"`      // 是否为公用
+	LastTime      string   `field:"lastTime"`      // 最新时间
 }
 
 type MetricItemOperator struct {
-	Id         interface{} // ID
-	IsOn       interface{} // 是否启用
-	Code       interface{} // 代号（用来区分是否内置）
-	Category   interface{} // 类型，比如http, tcp等
-	AdminId    interface{} // 管理员ID
-	UserId     interface{} // 用户ID
-	Name       interface{} // 指标名称
-	Keys       interface{} // 统计的Key
-	Period     interface{} // 周期
-	PeriodUnit interface{} // 周期单位
-	Value      interface{} // 值运算
-	State      interface{} // 状态
-	Version    interface{} // 版本号
-	IsPublic   interface{} // 是否为公用
+	Id            interface{} // ID
+	IsOn          interface{} // 是否启用
+	Code          interface{} // 代号（用来区分是否内置）
+	Category      interface{} // 类型，比如http, tcp等
+	AdminId       interface{} // 管理员ID
+	UserId        interface{} // 用户ID
+	Name          interface{} // 指标名称
+	Keys          interface{} // 统计的Key
+	Period        interface{} // 周期
+	PeriodUnit    interface{} // 周期单位
+	ExpiresPeriod interface{} // 过期周期
+	Value         interface{} // 值运算
+	State         interface{} // 状态
+	Version       interface{} // 版本号
+	IsPublic      interface{} // 是否为公用
+	LastTime      interface{} // 最新时间
 }
 
 func NewMetricItemOperator() *MetricItemOperator {

internal/db/models/metric_stat_dao.go

@@ -3,16 +3,21 @@ package models
 import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/lists"
-	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sort"
 	"strconv"
+	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -26,13 +31,15 @@ func init() {
 			for range ticker.C {
 				err := SharedMetricStatDAO.Clean(nil)
 				if err != nil {
-					logs.Println("SharedMetricStatDAO: clean expired data failed: " + err.Error())
+					remotelogs.Error("SharedMetricStatDAO", "clean expired data failed: "+err.Error())
 				}
 			}
 		})
 	})
 }
 
+const MetricStatTablePartials = 20 // 表格Partial数量
+
 func NewMetricStatDAO() *MetricStatDAO {
 	return dbs.NewDAO(&MetricStatDAO{
 		DAOObject: dbs.DAOObject{
@@ -65,7 +72,8 @@ func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64,
 	} else {
 		keysString = "[]"
 	}
-	return this.Query(tx).
+	err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Param("value", value).
 		InsertOrUpdateQuickly(maps.Map{
 			"hash":       hash,
@@ -81,57 +89,135 @@ func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64,
 		}, maps.Map{
 			"value": value,
 		})
+	if err != nil {
+		if this.canIgnore(err) {
+			return nil
+		}
+		return err
+	}
+
+	return SharedMetricItemDAO.UpdateMetricLastTime(tx, itemId, time)
 }
 
 // DeleteOldVersionItemStats 删除以前版本的统计数据
 func (this *MetricStatDAO) DeleteOldVersionItemStats(tx *dbs.Tx, itemId int64, version int32) error {
-	_, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Where("version<:version").
-		Param("version", version).
-		Delete()
-	return err
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Where("version<:version").
+			Param("version", version).
+			Delete()
+		return err
+	})
 }
 
 // DeleteItemStats 删除某个指标相关的统计数据
 func (this *MetricStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error {
-	_, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Delete()
-	return err
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Delete()
+		return err
+	})
 }
 
 // DeleteNodeItemStats 删除某个节点的统计数据
 func (this *MetricStatDAO) DeleteNodeItemStats(tx *dbs.Tx, nodeId int64, serverId int64, itemId int64, time string) error {
-	_, err := this.Query(tx).
-		Attr("nodeId", nodeId).
-		Attr("serverId", serverId).
-		Attr("itemId", itemId).
-		Attr("time", time).
-		Delete()
+	if serverId > 0 {
+		_, err := this.Query(tx).
+			Table(this.partialTable(serverId)).
+			Attr("nodeId", nodeId).
+			Attr("serverId", serverId).
+			Attr("itemId", itemId).
+			Attr("time", time).
+			Delete()
+		if this.canIgnore(err) {
+			return nil
+		}
+		return err
+	}
+
+	err := this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Attr("nodeId", nodeId).
+			Attr("serverId", serverId).
+			Attr("itemId", itemId).
+			Attr("time", time).
+			Delete()
+		return err
+	})
+
 	return err
 }
 
 // CountItemStats 计算统计数据数量
 func (this *MetricStatDAO) CountItemStats(tx *dbs.Tx, itemId int64, version int32) (int64, error) {
-	return this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Count()
+	var total int64 = 0
+
+	err := this.runBatch(func(table string, locker *sync.Mutex) error {
+		count, err := this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Count()
+		if err != nil {
+			return err
+		}
+		atomic.AddInt64(&total, count)
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
 }
 
 // ListItemStats 列出单页统计数据
 func (this *MetricStatDAO) ListItemStats(tx *dbs.Tx, itemId int64, version int32, offset int64, size int64) (result []*MetricStat, err error) {
-	_, err = this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Offset(offset).
-		Limit(size).
-		Desc("time").
-		Desc("serverId").
-		Desc("value").
-		Slice(&result).
-		FindAll()
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		_, err = this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Offset(offset).
+			Limit(size).
+			Desc("time").
+			Desc("serverId").
+			Desc("value").
+			Slice(&partialResult).
+			FindAll()
+		if err != nil {
+			return err
+		}
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].Time > result[j].Time {
+			return true
+		}
+		if result[i].ServerId > result[j].ServerId {
+			return true
+		}
+		if result[i].Value > result[j].Value {
+			return true
+		}
+		return false
+	})
+
 	return
 }
 
@@ -139,92 +225,127 @@ func (this *MetricStatDAO) ListItemStats(tx *dbs.Tx, itemId int64, version int32
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsAtLastTime(tx *dbs.Tx, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	// 最近一次时间
-	statOne, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		DescPk().
-		Find()
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
+	if err != nil || len(lastTime) == 0 {
+		return nil, err
+	}
+
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Attr("time", lastTime).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("MIN(time) AS time", "SUM(value) AS value", "keys").
+			Desc("value").
+			Group("keys").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+		}
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	if statOne == nil {
-		return nil, nil
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Value > result[j].Value
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
-	var lastStat = statOne.(*MetricStat)
-	var lastTime = lastStat.Time
-	var query = this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Attr("time", lastTime).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("MIN(time) AS time", "SUM(value) AS value", "keys").
-		Desc("value").
-		Group("keys").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
-		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
-	}
-	_, err = query.
-		FindAll()
+
 	return
 }
 
 // FindItemStatsWithClusterIdAndLastTime 取得集群最近一次计时前 N 个数据
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsWithClusterIdAndLastTime(tx *dbs.Tx, clusterId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	// 最近一次时间
-	statOne, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		DescPk().
-		Find()
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
+	if err != nil || len(lastTime) == 0 {
+		return nil, err
+	}
+
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			UseIndex("cluster_item_time").
+			Attr("clusterId", clusterId).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Attr("time", lastTime).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("MIN(time) AS time", "SUM(value) AS value", "keys").
+			Desc("value").
+			Group("keys").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+		}
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
-	if statOne == nil {
-		return nil, nil
-	}
-	var lastStat = statOne.(*MetricStat)
-	var lastTime = lastStat.Time
 
-	var query = this.Query(tx).
-		UseIndex("cluster_item_time").
-		Attr("clusterId", clusterId).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Attr("time", lastTime).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("MIN(time) AS time", "SUM(value) AS value", "keys").
-		Desc("value").
-		Group("keys").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
-		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Value > result[j].Value
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
 
-	_, err = query.
-		FindAll()
 	return
 }
 
@@ -232,68 +353,80 @@ func (this *MetricStatDAO) FindItemStatsWithClusterIdAndLastTime(tx *dbs.Tx, clu
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsWithNodeIdAndLastTime(tx *dbs.Tx, nodeId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	// 最近一次时间
-	statOne, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		DescPk().
-		Find()
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
 	if err != nil {
 		return nil, err
 	}
-	if statOne == nil {
-		return nil, nil
-	}
-	var lastStat = statOne.(*MetricStat)
-	var lastTime = lastStat.Time
-	var query = this.Query(tx).
-		UseIndex("node_item_time").
-		Attr("nodeId", nodeId).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Attr("time", lastTime).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("MIN(time) AS time", "SUM(value) AS value", "keys").
-		Desc("value").
-		Group("keys").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
+
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			UseIndex("node_item_time").
+			Attr("nodeId", nodeId).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Attr("time", lastTime).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("MIN(time) AS time", "SUM(value) AS value", "keys").
+			Desc("value").
+			Group("keys").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Value > result[j].Value
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
 
-	_, err = query.
-		FindAll()
 	return
 }
 
-// FindItemStatsWithServerIdAndLastTime 取得节点最近一次计时前 N 个数据
+// FindItemStatsWithServerIdAndLastTime 取得服务最近一次计时前 N 个数据
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsWithServerIdAndLastTime(tx *dbs.Tx, serverId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	// 最近一次时间
-	statOne, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		DescPk().
-		Find()
-	if err != nil {
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
+	if err != nil || len(lastTime) == 0 {
 		return nil, err
 	}
-	if statOne == nil {
-		return nil, nil
-	}
-	var lastStat = statOne.(*MetricStat)
-	var lastTime = lastStat.Time
 
 	var query = this.Query(tx).
+		Table(this.partialTable(serverId)).
 		UseIndex("server_item_time").
 		Attr("serverId", serverId).
 		Attr("itemId", itemId).
@@ -326,68 +459,119 @@ func (this *MetricStatDAO) FindItemStatsWithServerIdAndLastTime(tx *dbs.Tx, serv
 // FindLatestItemStats 取得所有集群上最近 N 个时间的数据
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStats(tx *dbs.Tx, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	var query = this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
-		Desc("time").
-		Group("time").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
+			Desc("time").
+			Group("time").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+		}
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
-	}
 
-	_, err = query.
-		FindAll()
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
 	if err != nil {
 		return nil, err
 	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Time > result[j].Time
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	lists.Reverse(result)
+
 	return
 }
 
 // FindLatestItemStatsWithClusterId 取得集群最近 N 个时间的数据
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStatsWithClusterId(tx *dbs.Tx, clusterId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	var query = this.Query(tx).
-		Attr("clusterId", clusterId).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
-		Desc("time").
-		Group("time").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			Attr("clusterId", clusterId).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
+			Desc("time").
+			Group("time").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+		}
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
-	}
 
-	_, err = query.
-		FindAll()
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
 	if err != nil {
 		return nil, err
 	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Time > result[j].Time
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	lists.Reverse(result)
 	return
 }
@@ -395,34 +579,55 @@ func (this *MetricStatDAO) FindLatestItemStatsWithClusterId(tx *dbs.Tx, clusterI
 // FindLatestItemStatsWithNodeId 取得节点最近 N 个时间的数据
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStatsWithNodeId(tx *dbs.Tx, nodeId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	var query = this.Query(tx).
-		Attr("nodeId", nodeId).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
-		Desc("time").
-		Group("time").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			Attr("nodeId", nodeId).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
+			Desc("time").
+			Group("time").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Time > result[j].Time
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
 
-	_, err = query.
-		FindAll()
-	if err != nil {
-		return nil, err
-	}
 	lists.Reverse(result)
 	return
 }
@@ -431,6 +636,7 @@ func (this *MetricStatDAO) FindLatestItemStatsWithNodeId(tx *dbs.Tx, nodeId int6
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStatsWithServerId(tx *dbs.Tx, serverId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	var query = this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Attr("serverId", serverId).
 		Attr("itemId", itemId).
 		Attr("version", version).
@@ -474,17 +680,21 @@ func (this *MetricStatDAO) Clean(tx *dbs.Tx) error {
 			}
 			for _, item := range items {
 				var config = &serverconfigs.MetricItemConfig{
-					Id:         int64(item.Id),
-					Period:     int(item.Period),
-					PeriodUnit: item.PeriodUnit,
+					Id:            int64(item.Id),
+					Period:        int(item.Period),
+					PeriodUnit:    item.PeriodUnit,
+					ExpiresPeriod: int(item.ExpiresPeriod),
 				}
 				var expiresDay = config.ServerExpiresDay()
-				_, err := this.Query(tx).
-					Attr("itemId", item.Id).
-					Lte("createdDay", expiresDay).
-					UseIndex("createdDay").
-					Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作
-					Delete()
+				err := this.runBatch(func(table string, locker *sync.Mutex) error {
+					_, err := this.Query(tx).
+						Table(table).
+						Attr("itemId", item.Id).
+						Lte("createdDay", expiresDay).
+						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
+						Delete()
+					return err
+				})
 				if err != nil {
 					return err
 				}
@@ -499,3 +709,60 @@ func (this *MetricStatDAO) Clean(tx *dbs.Tx) error {
 	}
 	return nil
 }
+
+// 获取分区表
+func (this *MetricStatDAO) partialTable(serverId int64) string {
+	return this.Table + "_" + types.String(serverId%int64(MetricStatTablePartials))
+}
+
+// 批量执行
+func (this *MetricStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
+	var locker = &sync.Mutex{}
+	var wg = sync.WaitGroup{}
+	wg.Add(MetricStatTablePartials)
+	var resultErr error
+	for i := 0; i < MetricStatTablePartials; i++ {
+		var table = this.partialTable(int64(i))
+		go func(table string) {
+			defer wg.Done()
+
+			err := f(table, locker)
+			if err != nil {
+				resultErr = err
+			}
+		}(table)
+	}
+	wg.Wait()
+	return resultErr
+}
+
+// 合并统计数据
+func (this *MetricStatDAO) mergeStats(stats []*MetricStat) (result []*MetricStat) {
+	var m = map[string]*MetricStat{} // key+time => *MetricStat
+	for _, stat := range stats {
+		var uniqueKey = stat.Time + "@" + string(stat.Keys)
+		oldStat, ok := m[uniqueKey]
+		if !ok {
+			result = append(result, stat)
+			m[uniqueKey] = stat
+		} else {
+			oldStat.Value += stat.Value
+		}
+	}
+	return result
+}
+
+// 检查错误是否可以忽略
+func (this *MetricStatDAO) canIgnore(err error) bool {
+	if err == nil {
+		return true
+	}
+
+	// 忽略 Error 1213: Deadlock found 错误
+	mysqlErr, ok := err.(*mysql.MySQLError)
+	if ok && mysqlErr.Number == 1213 {
+		return true
+	}
+
+	return false
+}

internal/db/models/metric_stat_dao_test.go

@@ -1,6 +1,7 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/dbs"
@@ -8,11 +9,12 @@ import (
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"testing"
+	"time"
 )
 
 func TestNewMetricStatDAO_InsertMany(t *testing.T) {
 	for i := 0; i <= 1; i++ {
-		err := NewMetricStatDAO().CreateStat(nil, types.String(i)+"_v1", 18, int64(rands.Int(0, 10000)), int64(rands.Int(0, 10000)), int64(rands.Int(0, 100)), []string{"/html" + types.String(i)}, 1, timeutil.Format("Ymd"), 0)
+		err := models.NewMetricStatDAO().CreateStat(nil, types.String(i)+"_v1", 18, int64(rands.Int(0, 10000)), int64(rands.Int(0, 10000)), int64(rands.Int(0, 100)), []string{"/html" + types.String(i)}, 1, timeutil.Format("Ymd"), 0)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -23,12 +25,38 @@ func TestNewMetricStatDAO_InsertMany(t *testing.T) {
 	t.Log("done")
 }
 
-func TestMetricStatDAO_Clean(t *testing.T) {
+func TestMetricStatDAO_Clean2(t *testing.T) {
 	dbs.NotifyReady()
 
-	err := NewMetricStatDAO().Clean(nil)
+	err := models.NewMetricStatDAO().Clean(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Log("ok")
 }
+
+func TestMetricStatDAO_DeleteNodeItemStats(t *testing.T) {
+	var dao = models.NewMetricStatDAO()
+	var before = time.Now()
+	defer func() {
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}()
+	err := dao.DeleteNodeItemStats(nil, 1, 0, 1, timeutil.Format("Ymd"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestMetricStatDAO_CountItemStats(t *testing.T) {
+	var dao = models.NewMetricStatDAO()
+	var before = time.Now()
+	defer func() {
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}()
+	count, err := dao.CountItemStats(nil, 1, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("count:", count)
+}

internal/db/models/metric_sum_stat_dao.go

@@ -2,19 +2,24 @@ package models
 
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
-	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sync"
 	"time"
 )
 
 type MetricSumStatDAO dbs.DAO
 
+const MetricSumStatTablePartials = 20 // 表格Partial数量
+
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
@@ -23,7 +28,7 @@ func init() {
 			for range ticker.C {
 				err := SharedMetricSumStatDAO.Clean(nil)
 				if err != nil {
-					logs.Println("SharedMetricSumStatDAO: clean expired data failed: " + err.Error())
+					remotelogs.Error("SharedMetricSumStatDAO", "clean expired data failed: "+err.Error())
 				}
 			}
 		})
@@ -51,7 +56,8 @@ func init() {
 
 // UpdateSum 更新统计数据
 func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, time string, itemId int64, version int32, count int64, total float32) error {
-	return this.Query(tx).
+	err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		InsertOrUpdateQuickly(maps.Map{
 			"clusterId":  clusterId,
 			"nodeId":     nodeId,
@@ -66,11 +72,16 @@ func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int6
 			"count": count,
 			"total": total,
 		})
+	if this.canIgnore(err) {
+		return nil
+	}
+	return err
 }
 
 // FindNodeServerSum 查找某个服务在某个节点上的统计数据
 func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
 	one, err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Attr("nodeId", nodeId).
 		Attr("serverId", serverId).
 		Attr("time", time).
@@ -83,29 +94,42 @@ func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, server
 	if one == nil {
 		return
 	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+
+	count = int64(one.(*MetricSumStat).Count)
+	total = float32(one.(*MetricSumStat).Total)
+
+	return
 }
 
 // FindSumAtTime 查找某个时间的统计数据
 func (this *MetricSumStatDAO) FindSumAtTime(tx *dbs.Tx, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
-		Attr("time", time).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
-		Find()
-	if err != nil {
-		return 0, 0, err
-	}
-	if one == nil {
-		return
-	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		one, err := this.Query(tx).
+			Table(table).
+			Attr("time", time).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Result("SUM(count) AS `count`, SUM(total) AS total").
+			Find()
+		if err != nil {
+			return err
+		}
+		if one == nil {
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+	return
 }
 
 // FindServerSum 查找某个服务的统计数据
 func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
 	one, err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		UseIndex("server_item_time").
 		Attr("serverId", serverId).
 		Attr("time", time).
@@ -124,48 +148,69 @@ func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time str
 
 // FindClusterSum 查找集群上的统计数据
 func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
-		UseIndex("cluster_item_time").
-		Attr("clusterId", clusterId).
-		Attr("time", time).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
-		Find()
-	if err != nil {
-		return 0, 0, err
-	}
-	if one == nil {
-		return
-	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		one, err := this.Query(tx).
+			Table(table).
+			UseIndex("cluster_item_time").
+			Attr("clusterId", clusterId).
+			Attr("time", time).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Result("SUM(count) AS `count`, SUM(total) AS total").
+			Find()
+		if err != nil {
+			return err
+		}
+		if one == nil {
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+
+	return
 }
 
 // FindNodeSum 查找节点上的统计数据
 func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
-		UseIndex("node_item_time").
-		Attr("nodeId", nodeId).
-		Attr("time", time).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
-		Find()
-	if err != nil {
-		return 0, 0, err
-	}
-	if one == nil {
-		return
-	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		one, err := this.Query(tx).
+			Table(table).
+			UseIndex("node_item_time").
+			Attr("nodeId", nodeId).
+			Attr("time", time).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Result("SUM(count) AS `count`, SUM(total) AS total").
+			Find()
+		if err != nil {
+			return err
+		}
+		if one == nil {
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+
+	return
 }
 
 // DeleteItemStats 删除某个指标相关的统计数据
 func (this *MetricSumStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error {
-	_, err := this.Query(tx).
-		Attr("itemId", itemId).
-		Delete()
-	return err
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Delete()
+		return err
+	})
 }
 
 // Clean 清理数据
@@ -180,18 +225,22 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 			}
 			for _, item := range items {
 				var config = &serverconfigs.MetricItemConfig{
-					Id:         int64(item.Id),
-					Period:     int(item.Period),
-					PeriodUnit: item.PeriodUnit,
+					Id:            int64(item.Id),
+					Period:        int(item.Period),
+					PeriodUnit:    item.PeriodUnit,
+					ExpiresPeriod: int(item.ExpiresPeriod),
 				}
 				var expiresDay = config.ServerExpiresDay()
-				_, err := this.Query(tx).
-					Attr("itemId", item.Id).
-					Where("(createdDay IS NULL OR createdDay<:day)").
-					Param("day", expiresDay).
-					UseIndex("createdDay").
-					Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作
-					Delete()
+				err = this.runBatch(func(table string, locker *sync.Mutex) error {
+					_, err := this.Query(tx).
+						Table(table).
+						Attr("itemId", item.Id).
+						Where("(createdDay IS NULL OR createdDay<:day)").
+						Param("day", expiresDay).
+						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
+						Delete()
+					return err
+				})
 				if err != nil {
 					return err
 				}
@@ -206,3 +255,44 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 	}
 	return nil
 }
+
+// 获取分区表
+func (this *MetricSumStatDAO) partialTable(serverId int64) string {
+	return this.Table + "_" + types.String(serverId%int64(MetricSumStatTablePartials))
+}
+
+// 批量执行
+func (this *MetricSumStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
+	var locker = &sync.Mutex{}
+	var wg = sync.WaitGroup{}
+	wg.Add(MetricSumStatTablePartials)
+	var resultErr error
+	for i := 0; i < MetricSumStatTablePartials; i++ {
+		var table = this.partialTable(int64(i))
+		go func(table string) {
+			defer wg.Done()
+
+			err := f(table, locker)
+			if err != nil {
+				resultErr = err
+			}
+		}(table)
+	}
+	wg.Wait()
+	return resultErr
+}
+
+// 检查错误是否可以忽略
+func (this *MetricSumStatDAO) canIgnore(err error) bool {
+	if err == nil {
+		return true
+	}
+
+	// 忽略 Error 1213: Deadlock found 错误
+	mysqlErr, ok := err.(*mysql.MySQLError)
+	if ok && mysqlErr.Number == 1213 {
+		return true
+	}
+
+	return false
+}