提交SQL

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2020, LiuXiangChao
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1 @@
+GoEdge API节点源码

build/build.sh

@@ -1,76 +1,76 @@
 #!/usr/bin/env bash
 
 function build() {
-	ROOT=$(dirname $0)
+	ROOT=$(dirname "$0")
 	NAME="edge-api"
 	DIST=$ROOT/"../dist/${NAME}"
 	OS=${1}
 	ARCH=${2}
 	TAG=${3}
-	NODE_ARCHITECTS=("amd64" "386" "arm64" "mips64" "mips64le")
+	NODE_ARCHITECTS=("amd64" "arm64")
 
-	if [ -z $OS ]; then
+	if [ -z "$OS" ]; then
 		echo "usage: build.sh OS ARCH"
 		exit
 	fi
-	if [ -z $ARCH ]; then
+	if [ -z "$ARCH" ]; then
 		echo "usage: build.sh OS ARCH"
 		exit
 	fi
-	if [ -z $TAG ]; then
+	if [ -z "$TAG" ]; then
 		TAG="community"
 	fi
 
-	VERSION=$(lookup-version $ROOT/../internal/const/const.go)
+	VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
 	ZIP="${NAME}-${OS}-${ARCH}-${TAG}-v${VERSION}.zip"
 
 	# build edge-node
-	NodeVersion=$(lookup-version $ROOT"/../../EdgeNode/internal/const/const.go")
+	NodeVersion=$(lookup-version "$ROOT""/../../EdgeNode/internal/const/const.go")
 	echo "building edge-node v${NodeVersion} ..."
 	EDGE_NODE_BUILD_SCRIPT=$ROOT"/../../EdgeNode/build/build.sh"
-	if [ ! -f $EDGE_NODE_BUILD_SCRIPT ]; then
+	if [ ! -f "$EDGE_NODE_BUILD_SCRIPT" ]; then
 		echo "unable to find edge-node build script 'EdgeNode/build/build.sh'"
 		exit
 	fi
-	cd $ROOT"/../../EdgeNode/build"
+	cd "$ROOT""/../../EdgeNode/build" || exit
 	echo "=============================="
 	for arch in "${NODE_ARCHITECTS[@]}"; do
-		if [ ! -f $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" ]; then
+		if [ ! -f "$ROOT""/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" ]; then
-			./build.sh linux $arch $TAG
+			./build.sh linux "$arch" $TAG
 		else
 			echo "use built node linux/$arch/v${NodeVersion}"
 		fi
 	done
 	echo "=============================="
-	cd -
+	cd - || exit
 
-	rm -f $ROOT/deploy/*.zip
+	rm -f "$ROOT"/deploy/*.zip
 	for arch in "${NODE_ARCHITECTS[@]}"; do
-		cp $ROOT"/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" $ROOT/deploy/edge-node-linux-${arch}-v${NodeVersion}.zip
+		cp "$ROOT""/../../EdgeNode/dist/edge-node-linux-${arch}-${TAG}-v${NodeVersion}.zip" "$ROOT"/deploy/edge-node-linux-"${arch}"-v"${NodeVersion}".zip
 	done
 
 	# build edge-dns
 	if [ "$TAG" = "plus" ]; then
 		DNS_ROOT=$ROOT"/../../EdgeDNS"
-		if [ -d $DNS_ROOT  ]; then
+		if [ -d "$DNS_ROOT"  ]; then
-			DNSNodeVersion=$(lookup-version $ROOT"/../../EdgeDNS/internal/const/const.go")
+			DNSNodeVersion=$(lookup-version "$ROOT""/../../EdgeDNS/internal/const/const.go")
 			echo "building edge-dns ${DNSNodeVersion} ..."
 			EDGE_DNS_NODE_BUILD_SCRIPT=$ROOT"/../../EdgeDNS/build/build.sh"
-			if [ ! -f $EDGE_DNS_NODE_BUILD_SCRIPT ]; then
+			if [ ! -f "$EDGE_DNS_NODE_BUILD_SCRIPT" ]; then
 				echo "unable to find edge-dns build script 'EdgeDNS/build/build.sh'"
 				exit
 			fi
-			cd $ROOT"/../../EdgeDNS/build"
+			cd "$ROOT""/../../EdgeDNS/build" || exit
 			echo "=============================="
-			architects=("amd64")
+			architects=("amd64" "arm64")
 			for arch in "${architects[@]}"; do
-				./build.sh linux $arch $TAG
+				./build.sh linux "$arch" $TAG
 			done
 			echo "=============================="
-			cd -
+			cd - || exit
 
 			for arch in "${architects[@]}"; do
-				cp $ROOT"/../../EdgeDNS/dist/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip" $ROOT/deploy/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip
+				cp "$ROOT""/../../EdgeDNS/dist/edge-dns-linux-${arch}-v${DNSNodeVersion}.zip" "$ROOT"/deploy/edge-dns-linux-"${arch}"-v"${DNSNodeVersion}".zip
 			done
 		fi
 	fi
@@ -78,48 +78,48 @@ function build() {
 	# build sql
 	if [ $TAG = "plus" ]; then
 		echo "building sql ..."
-		${ROOT}/sql.sh
+		"${ROOT}"/sql.sh
 	fi
 
 	# copy files
 	echo "copying ..."
-	if [ ! -d $DIST ]; then
+	if [ ! -d "$DIST" ]; then
-		mkdir $DIST
+		mkdir "$DIST"
-		mkdir $DIST/bin
+		mkdir "$DIST"/bin
-		mkdir $DIST/configs
+		mkdir "$DIST"/configs
-		mkdir $DIST/logs
+		mkdir "$DIST"/logs
 	fi
-	cp $ROOT/configs/api.template.yaml $DIST/configs/
+	cp "$ROOT"/configs/api.template.yaml "$DIST"/configs/
-	cp $ROOT/configs/db.template.yaml $DIST/configs/
+	cp "$ROOT"/configs/db.template.yaml "$DIST"/configs/
-	cp -R $ROOT/deploy $DIST/
+	cp -R "$ROOT"/deploy "$DIST/"
-	rm -f $dist/deploy/.gitignore
+	rm -f "$DIST"/deploy/.gitignore
-	cp -R $ROOT/installers $DIST/
+	cp -R "$ROOT"/installers "$DIST"/
-	cp -R $ROOT/resources $DIST/
+	cp -R "$ROOT"/resources "$DIST"/
-	rm -f $DIST/resources/ipdata/ip2region/global_region.csv
+	rm -f "$DIST"/resources/ipdata/ip2region/global_region.csv
-	rm -f $DIST/resources/ipdata/ip2region/ip.merge.txt
+	rm -f "$DIST"/resources/ipdata/ip2region/ip.merge.txt
 
 	# building edge installer
 	echo "building node installer ..."
-	architects=("amd64" "386" "arm64")
+	architects=("amd64" "arm64")
 	for arch in "${architects[@]}"; do
 		# TODO support arm, mips ...
-		env GOOS=linux GOARCH=${arch} go build -tags $TAG --ldflags="-s -w" -o $ROOT/installers/edge-installer-helper-linux-${arch} $ROOT/../cmd/installer-helper/main.go
+		env GOOS=linux GOARCH="${arch}" go build -trimpath -tags $TAG --ldflags="-s -w" -o "$ROOT"/installers/edge-installer-helper-linux-"${arch}" "$ROOT"/../cmd/installer-helper/main.go
 	done
 
 	# building edge dns installer
 	echo "building dns node installer ..."
-	architects=("amd64" "386" "arm64")
+	architects=("amd64" "arm64")
 	for arch in "${architects[@]}"; do
 		# TODO support arm, mips ...
-		env GOOS=linux GOARCH=${arch} go build -tags $TAG --ldflags="-s -w" -o $ROOT/installers/edge-installer-dns-helper-linux-${arch} $ROOT/../cmd/installer-dns-helper/main.go
+		env GOOS=linux GOARCH="${arch}" go build -trimpath -tags $TAG --ldflags="-s -w" -o "$ROOT"/installers/edge-installer-dns-helper-linux-"${arch}" "$ROOT"/../cmd/installer-dns-helper/main.go
 	done
 
 	# building api node
-	env GOOS=$OS GOARCH=$ARCH go build -tags $TAG --ldflags="-s -w" -o $DIST/bin/edge-api $ROOT/../cmd/edge-api/main.go
+	env GOOS="$OS" GOARCH="$ARCH" go build -trimpath -tags $TAG --ldflags="-s -w" -o "$DIST"/bin/edge-api "$ROOT"/../cmd/edge-api/main.go
 
 	# delete hidden files
-	find $DIST -name ".DS_Store" -delete
+	find "$DIST" -name ".DS_Store" -delete
-	find $DIST -name ".gitignore" -delete
+	find "$DIST" -name ".gitignore" -delete
 
 	echo "zip files"
 	cd "${DIST}/../" || exit
@@ -135,15 +135,15 @@ function build() {
 
 function lookup-version() {
 	FILE=$1
-	VERSION_DATA=$(cat $FILE)
+	VERSION_DATA=$(cat "$FILE")
 	re="Version[ ]+=[ ]+\"([0-9.]+)\""
 	if [[ $VERSION_DATA =~ $re ]]; then
 		VERSION=${BASH_REMATCH[1]}
-		echo $VERSION
+		echo "$VERSION"
 	else
 		echo "could not match version"
 		exit
 	fi
 }
 
-build $1 $2 $3
+build "$1" "$2" "$3"

cmd/edge-api/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"github.com/TeaOSLab/EdgeAPI/internal/apps"
 	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
@@ -11,7 +12,9 @@ import (
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/types"
 	"github.com/iwind/gosock/pkg/gosock"
+	"io/ioutil"
 	"log"
 	"os"
 )
@@ -20,12 +23,12 @@ func main() {
 	if !Tea.IsTesting() {
 		Tea.Env = "prod"
 	}
-	app := apps.NewAppCmd()
+	var app = apps.NewAppCmd()
 	app.Version(teaconst.Version)
 	app.Product(teaconst.ProductName)
-	app.Usage(teaconst.ProcessName + " [start|stop|restart|setup|upgrade|service|daemon]")
+	app.Usage(teaconst.ProcessName + " [start|stop|restart|setup|upgrade|service|daemon|issues]")
 	app.On("setup", func() {
-		setupCmd := setup.NewSetupFromCmd()
+		var setupCmd = setup.NewSetupFromCmd()
 		err := setupCmd.Run()
 		result := maps.Map{}
 		if err != nil {
@@ -97,6 +100,68 @@ func main() {
 			}
 		}
 	})
+	app.On("db.stmt.prepare", func() {
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{Code: "db.stmt.prepare"})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+		} else {
+			var isOn = maps.NewMap(reply.Params).GetBool("isOn")
+			if isOn {
+				fmt.Println("show statements: on")
+			} else {
+				fmt.Println("show statements: off")
+			}
+		}
+	})
+	app.On("db.stmt.count", func() {
+		var sock = gosock.NewTmpSock(teaconst.ProcessName)
+		reply, err := sock.Send(&gosock.Command{Code: "db.stmt.count"})
+		if err != nil {
+			fmt.Println("[ERROR]" + err.Error())
+		} else {
+			var count = maps.NewMap(reply.Params).GetInt("count")
+			fmt.Println("prepared statements count: " + types.String(count))
+		}
+	})
+	app.On("issues", func() {
+		var flagSet = flag.NewFlagSet("issues", flag.ExitOnError)
+		var formatJSON = false
+		flagSet.BoolVar(&formatJSON, "json", false, "")
+		_ = flagSet.Parse(os.Args[2:])
+
+		data, err := ioutil.ReadFile(Tea.LogFile("issues.log"))
+		if err != nil {
+			if formatJSON {
+				fmt.Print("[]")
+			} else {
+				fmt.Println("no issues yet")
+			}
+		} else {
+			var issueMaps = []maps.Map{}
+			err = json.Unmarshal(data, &issueMaps)
+			if err != nil {
+				if formatJSON {
+					fmt.Print("[]")
+				} else {
+					fmt.Println("no issues yet")
+				}
+			} else {
+				if formatJSON {
+					fmt.Print(string(data))
+				} else {
+					if len(issueMaps) == 0 {
+						fmt.Println("no issues yet")
+					} else {
+						for i, issue := range issueMaps {
+							fmt.Println("issue " + types.String(i+1) + ": " + issue.GetString("message"))
+						}
+					}
+				}
+			}
+		}
+	})
+
 	app.Run(func() {
 		nodes.NewAPINode().Start()
 	})

dist/.gitignore

@@ -1 +1,2 @@
 *.zip
+edge-api

go.mod

@@ -1,6 +1,6 @@
 module github.com/TeaOSLab/EdgeAPI
 
-go 1.16
+go 1.18
 
 replace github.com/TeaOSLab/EdgeCommon => ../EdgeCommon
 
@@ -12,9 +12,8 @@ require (
 	github.com/go-acme/lego/v4 v4.5.2
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang/protobuf v1.5.2
-	github.com/iwind/TeaGo v0.0.0-20220321132348-7da816422f25 // indirect
+	github.com/iwind/TeaGo v0.0.0-20220408111647-f36b9bba3570
 	github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3
-	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mozillazg/go-pinyin v0.18.0
 	github.com/pkg/sftp v1.12.0
 	github.com/shirou/gopsutil/v3 v3.22.2
@@ -24,3 +23,25 @@ require (
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
+
+require (
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/tklauser/go-sysconf v0.3.9 // indirect
+	github.com/tklauser/numcpus v0.3.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+)

go.sum

@@ -66,7 +66,6 @@ github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7F
 github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -238,12 +237,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/infobloxopen/infoblox-go-client v1.1.1/go.mod h1:BXiw7S2b9qJoM8MS40vfgCNB2NLHGusk1DtO16BD9zI=
 github.com/iwind/TeaGo v0.0.0-20210411134150-ddf57e240c2f/go.mod h1:KU4mS7QNiZ7QWEuDBk1zw0/Q2LrAPZv3tycEFBsuUwc=
 github.com/iwind/TeaGo v0.0.0-20220304043459-0dd944a5b475/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
-github.com/iwind/TeaGo v0.0.0-20220321112016-5a2cd71d3151 h1:jksmjwlGC8QMpyHZmzxr7J+3NeMOr9Zy2+yNJxVSIjI=
+github.com/iwind/TeaGo v0.0.0-20220408111647-f36b9bba3570 h1:zqz2FiMMkSHXWO1EsTRJDPTwX9xQ4uuyD5GAE4JGlhM=
-github.com/iwind/TeaGo v0.0.0-20220321112016-5a2cd71d3151/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
+github.com/iwind/TeaGo v0.0.0-20220408111647-f36b9bba3570/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
-github.com/iwind/TeaGo v0.0.0-20220321131553-fd7b112ba7e7 h1:gdMQZQk/aXfNuKuWCBQhP3byy5Dr8XHMe5+GXdGHcPQ=
-github.com/iwind/TeaGo v0.0.0-20220321131553-fd7b112ba7e7/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
-github.com/iwind/TeaGo v0.0.0-20220321132348-7da816422f25 h1:UpJ52iq8FEz2OeaXFhW1kuYeqVRUQ/5N+NVHvVuTnvw=
-github.com/iwind/TeaGo v0.0.0-20220321132348-7da816422f25/go.mod h1:HRHK0zoC/og3c9/hKosD9yYVMTnnzm3PgXUdhRYHaLc=
 github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3 h1:aBSonas7vFcgTj9u96/bWGILGv1ZbUSTLiOzcI1ZT6c=
 github.com/iwind/gosock v0.0.0-20210722083328-12b2d66abec3/go.mod h1:H5Q7SXwbx3a97ecJkaS2sD77gspzE7HFUafBO0peEyA=
 github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
@@ -561,7 +556,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

internal/accesslogs/storage_base.go

@@ -12,8 +12,9 @@ import (
 )
 
 type BaseStorage struct {
-	isOk    bool
+	isOk         bool
-	version int
+	version      int
+	firewallOnly bool
 }
 
 func (this *BaseStorage) SetVersion(version int) {
@@ -32,6 +33,10 @@ func (this *BaseStorage) SetOk(isOk bool) {
 	this.isOk = isOk
 }
 
+func (this *BaseStorage) SetFirewallOnly(firewallOnly bool) {
+	this.firewallOnly = firewallOnly
+}
+
 // Marshal 对日志进行编码
 func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) {
 	return json.Marshal(accessLog)
@@ -39,7 +44,7 @@ func (this *BaseStorage) Marshal(accessLog *pb.HTTPAccessLog) ([]byte, error) {
 
 // FormatVariables 格式化字符串中的变量
 func (this *BaseStorage) FormatVariables(s string) string {
-	now := time.Now()
+	var now = time.Now()
 	return configutils.ParseVariables(s, func(varName string) (value string) {
 		switch varName {
 		case "year":

internal/accesslogs/storage_command.go

@@ -61,6 +61,10 @@ func (this *CommandStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 		return err
 	}
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
+
 		data, err := this.Marshal(accessLog)
 		if err != nil {
 			logs.Error(err)

internal/accesslogs/storage_es.go

@@ -59,6 +59,10 @@ func (this *ESStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	indexName := this.FormatVariables(this.config.Index)
 	typeName := this.FormatVariables(this.config.MappingType)
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
+
 		if len(accessLog.RequestId) == 0 {
 			continue
 		}

internal/accesslogs/storage_file.go

@@ -57,6 +57,9 @@ func (this *FileStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	defer this.writeLocker.Unlock()
 
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
 		data, err := this.Marshal(accessLog)
 		if err != nil {
 			logs.Error(err)

internal/accesslogs/storage_interface.go

@@ -12,6 +12,9 @@ type StorageInterface interface {
 	// SetVersion 设置版本
 	SetVersion(version int)
 
+	// SetFirewallOnly 设置是否只处理防火墙相关的访问日志
+	SetFirewallOnly(firewallOnly bool)
+
 	IsOk() bool
 
 	SetOk(ok bool)

internal/accesslogs/storage_manager.go

@@ -101,6 +101,7 @@ func (this *StorageManager) Loop() error {
 				}
 
 				storage.SetVersion(types.Int(policy.Version))
+				storage.SetFirewallOnly(policy.FirewallOnly == 1)
 				err := storage.Start()
 				if err != nil {
 					remotelogs.Error("ACCESS_LOG_STORAGE_MANAGER", "start policy '"+types.String(policyId)+"' failed: "+err.Error())
@@ -116,6 +117,7 @@ func (this *StorageManager) Loop() error {
 				continue
 			}
 			storage.SetVersion(types.Int(policy.Version))
+			storage.SetFirewallOnly(policy.FirewallOnly == 1)
 			this.storageMap[policyId] = storage
 			err = storage.Start()
 			if err != nil {

internal/accesslogs/storage_syslog.go

@@ -1,7 +1,9 @@
 package accesslogs
 
 import (
+	"bytes"
 	"errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/iwind/TeaGo/logs"
@@ -95,7 +97,10 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 
 	args = append(args, "-S", "10240")
 
-	cmd := exec.Command(this.exe, args...)
+	var cmd = exec.Command(this.exe, args...)
+	var stderrBuffer = &bytes.Buffer{}
+	cmd.Stderr = stderrBuffer
+
 	w, err := cmd.StdinPipe()
 	if err != nil {
 		return err
@@ -106,9 +111,12 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	}
 
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
 		data, err := this.Marshal(accessLog)
 		if err != nil {
-			logs.Error(err)
+			remotelogs.Error("ACCESS_LOG_POLICY_SYSLOG", "marshal accesslog failed: "+err.Error())
 			continue
 		}
 		_, err = w.Write(data)
@@ -118,14 +126,15 @@ func (this *SyslogStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 
 		_, err = w.Write([]byte("\n"))
 		if err != nil {
-			logs.Error(err)
+			remotelogs.Error("ACCESS_LOG_POLICY_SYSLOG", "write accesslog failed: "+err.Error())
 		}
 	}
 
 	_ = w.Close()
+
 	err = cmd.Wait()
 	if err != nil {
-		return err
+		return errors.New("send syslog failed: " + err.Error() + ", stderr: " + stderrBuffer.String())
 	}
 
 	return nil

internal/accesslogs/storage_tcp.go

@@ -60,6 +60,9 @@ func (this *TCPStorage) Write(accessLogs []*pb.HTTPAccessLog) error {
 	defer this.writeLocker.Unlock()
 
 	for _, accessLog := range accessLogs {
+		if this.firewallOnly && accessLog.FirewallPolicyId == 0 {
+			continue
+		}
 		data, err := this.Marshal(accessLog)
 		if err != nil {
 			logs.Error(err)

internal/apps/log_writer.go

@@ -1,51 +1,108 @@
 package apps
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/sizes"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/files"
-	"github.com/iwind/TeaGo/logs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
-	"github.com/iwind/TeaGo/utils/time"
 	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
 )
 
 type LogWriter struct {
-	fileAppender *files.Appender
+	fp *os.File
+	c  chan string
 }
 
 func (this *LogWriter) Init() {
 	// 创建目录
-	dir := files.NewFile(Tea.LogDir())
+	var dir = files.NewFile(Tea.LogDir())
 	if !dir.Exists() {
 		err := dir.Mkdir()
 		if err != nil {
-			log.Println("[error]" + err.Error())
+			log.Println("[LOG]create log dir failed: " + err.Error())
 		}
 	}
 
-	logFile := files.NewFile(Tea.LogFile("run.log"))
-
 	// 打开要写入的日志文件
-	appender, err := logFile.Appender()
+	var logPath = Tea.LogFile("run.log")
+	fp, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 	if err != nil {
-		logs.Error(err)
+		log.Println("[LOG]open log file failed: " + err.Error())
 	} else {
-		this.fileAppender = appender
+		this.fp = fp
+	}
+
+	this.c = make(chan string, 1024)
+
+	// 异步写入文件
+	var maxFileSize = 2 * sizes.G // 文件最大尺寸，超出此尺寸则清空
+	if fp != nil {
+		goman.New(func() {
+			var totalSize int64 = 0
+			stat, err := fp.Stat()
+			if err == nil {
+				totalSize = stat.Size()
+			}
+
+			for message := range this.c {
+				totalSize += int64(len(message))
+				_, err := fp.WriteString(timeutil.Format("Y/m/d H:i:s ") + message + "\n")
+				if err != nil {
+					log.Println("[LOG]write log failed: " + err.Error())
+				} else {
+					// 如果太大则Truncate
+					if totalSize > maxFileSize {
+						_ = fp.Truncate(0)
+						totalSize = 0
+					}
+				}
+			}
+		})
 	}
 }
 
 func (this *LogWriter) Write(message string) {
-	log.Println(message)
+	backgroundEnv, _ := os.LookupEnv("EdgeBackground")
+	if backgroundEnv != "on" {
+		// 文件和行号
+		var file string
+		var line int
+		if Tea.IsTesting() {
+			var callDepth = 3
+			var ok bool
+			_, file, line, ok = runtime.Caller(callDepth)
+			if ok {
+				file = this.packagePath(file)
+			}
+		}
 
-	if this.fileAppender != nil {
+		if len(file) > 0 {
-		_, err := this.fileAppender.AppendString(timeutil.Format("Y/m/d H:i:s ") + message + "\n")
+			log.Println(message + " (" + file + ":" + strconv.Itoa(line) + ")")
-		if err != nil {
+		} else {
-			log.Println("[error]" + err.Error())
+			log.Println(message)
 		}
 	}
+
+	this.c <- message
 }
 
 func (this *LogWriter) Close() {
-	if this.fileAppender != nil {
+	if this.fp != nil {
-		_ = this.fileAppender.Close()
+		_ = this.fp.Close()
 	}
+
+	close(this.c)
+}
+
+func (this *LogWriter) packagePath(path string) string {
+	var pieces = strings.Split(path, "/")
+	if len(pieces) >= 2 {
+		return strings.Join(pieces[len(pieces)-2:], "/")
+	}
+	return path
 }

internal/const/const.go

@@ -1,7 +1,7 @@
 package teaconst
 
 const (
-	Version = "0.4.6"
+	Version = "0.4.9"
 
 	ProductName   = "Edge API"
 	ProcessName   = "edge-api"
@@ -18,13 +18,13 @@ const (
 
 	// 其他节点版本号，用来检测是否有需要升级的节点
 
-	NodeVersion          = "0.4.5"
+	NodeVersion          = "0.4.9"
-	UserNodeVersion      = "0.3.2"
+	UserNodeVersion      = "0.3.5"
 	AuthorityNodeVersion = "0.0.2"
-	MonitorNodeVersion   = "0.0.3"
+	MonitorNodeVersion   = "0.0.4"
-	DNSNodeVersion       = "0.2.1"
+	DNSNodeVersion       = "0.2.4"
-	ReportNodeVersion    = "0.1.0"
+	ReportNodeVersion    = "0.1.1"
 
 	// SQLVersion SQL版本号
-	SQLVersion = "5"
+	SQLVersion = "2"
 )

internal/db/db_test.go

@@ -3,6 +3,7 @@ package db
 import (
 	"database/sql"
 	"database/sql/driver"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	_ "github.com/iwind/TeaGo/bootstrap"
@@ -42,3 +43,13 @@ func TestDB_Instance(t *testing.T) {
 	}
 	time.Sleep(100 * time.Second)
 }
+
+func TestDB_Reuse(t *testing.T) {
+	var dao = models.NewVersionDAO()
+	for i := 0; i < 20_000; i++ {
+		_, _, err := dao.Query(nil).Attr("version", i).Reuse(true).FindOne()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}

internal/db/models/accounts/user_account_log_dao.go

@@ -2,6 +2,7 @@ package accounts
 
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/userconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -79,7 +80,7 @@ func (this *UserAccountLogDAO) CountAccountLogs(tx *dbs.Tx, userId int64, accoun
 	}
 	if len(keyword) > 0 {
 		query.Where("(userId IN (SELECT id FROM " + models.SharedUserDAO.Table + " WHERE state=1 AND (username LIKE :keyword OR fullname LIKE :keyword)) OR description LIKE :keyword)")
-		query.Param("keyword", "%"+keyword+"%")
+		query.Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(eventType) > 0 {
 		query.Attr("eventType", eventType)
@@ -98,7 +99,7 @@ func (this *UserAccountLogDAO) ListAccountLogs(tx *dbs.Tx, userId int64, account
 	}
 	if len(keyword) > 0 {
 		query.Where("(userId IN (SELECT id FROM " + models.SharedUserDAO.Table + " WHERE state=1 AND (username LIKE :keyword OR fullname LIKE :keyword)) OR description LIKE :keyword)")
-		query.Param("keyword", "%"+keyword+"%")
+		query.Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(eventType) > 0 {
 		query.Attr("eventType", eventType)

internal/db/models/acme/acme_task_dao.go

@@ -125,11 +125,11 @@ func (this *ACMETaskDAO) CountAllEnabledACMETasks(tx *dbs.Tx, adminId int64, use
 
 	if len(keyword) > 0 {
 		query.Where("(domains LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(keyword) > 0 {
 		query.Where("domains LIKE :keyword").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	return query.State(ACMETaskStateEnabled).
@@ -155,7 +155,7 @@ func (this *ACMETaskDAO) ListEnabledACMETasks(tx *dbs.Tx, adminId int64, userId
 	}
 	if len(keyword) > 0 {
 		query.Where("(domains LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(ACMETaskStateEnabled).

internal/db/models/api_node_dao.go

@@ -3,6 +3,8 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/configs"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -49,16 +51,29 @@ func (this *APINodeDAO) EnableAPINode(tx *dbs.Tx, id int64) error {
 		Pk(id).
 		Set("state", APINodeStateEnabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+	return this.NotifyUpdate(tx, id)
 }
 
 // DisableAPINode 禁用条目
-func (this *APINodeDAO) DisableAPINode(tx *dbs.Tx, id int64) error {
+func (this *APINodeDAO) DisableAPINode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", APINodeStateDisabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+
+	err = this.NotifyUpdate(tx, nodeId)
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleAPI, nodeId)
 }
 
 // FindEnabledAPINode 查找启用中的条目
@@ -149,16 +164,33 @@ func (this *APINodeDAO) CreateAPINode(tx *dbs.Tx, name string, description strin
 		return
 	}
 
+	err = this.NotifyUpdate(tx, types.Int64(op.Id))
+	if err != nil {
+		remotelogs.Error("API_NODE_DAO", err.Error())
+	}
+
 	return types.Int64(op.Id), nil
 }
 
 // UpdateAPINode 修改API节点
-func (this *APINodeDAO) UpdateAPINode(tx *dbs.Tx, nodeId int64, name string, description string, httpJSON []byte, httpsJSON []byte, restIsOn bool, restHTTPJSON []byte, restHTTPSJSON []byte, accessAddrsJSON []byte, isOn bool) error {
+func (this *APINodeDAO) UpdateAPINode(tx *dbs.Tx, nodeId int64, name string, description string, httpJSON []byte, httpsJSON []byte, restIsOn bool, restHTTPJSON []byte, restHTTPSJSON []byte, accessAddrsJSON []byte, isOn bool, isPrimary bool) error {
 	if nodeId <= 0 {
 		return errors.New("invalid nodeId")
 	}
 
-	op := NewAPINodeOperator()
+	// 取消别的Primary
+	if isPrimary {
+		err := this.Query(tx).
+			Neq("id", nodeId).
+			Attr("isPrimary", true).
+			Set("isPrimary", false).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+	}
+
+	var op = NewAPINodeOperator()
 	op.Id = nodeId
 	op.Name = name
 	op.Description = description
@@ -191,8 +223,13 @@ func (this *APINodeDAO) UpdateAPINode(tx *dbs.Tx, nodeId int64, name string, des
 		op.AccessAddrs = "[]"
 	}
 
+	op.IsPrimary = isPrimary
+
 	err := this.Save(tx, op)
-	return err
+	if err != nil {
+		return err
+	}
+	return this.NotifyUpdate(tx, nodeId)
 }
 
 // FindAllEnabledAPINodes 列出所有可用API节点
@@ -294,23 +331,6 @@ func (this *APINodeDAO) UpdateAPINodeStatus(tx *dbs.Tx, apiNodeId int64, statusJ
 	return err
 }
 
-// 生成唯一ID
-func (this *APINodeDAO) genUniqueId(tx *dbs.Tx) (string, error) {
-	for {
-		uniqueId := rands.HexString(32)
-		ok, err := this.Query(tx).
-			Attr("uniqueId", uniqueId).
-			Exist()
-		if err != nil {
-			return "", err
-		}
-		if ok {
-			continue
-		}
-		return uniqueId, nil
-	}
-}
-
 // CountAllLowerVersionNodes 计算所有节点中低于某个版本的节点数量
 func (this *APINodeDAO) CountAllLowerVersionNodes(tx *dbs.Tx, version string) (int64, error) {
 	return this.Query(tx).
@@ -384,3 +404,114 @@ func (this *APINodeDAO) FindAllEnabledAPIAccessIPs(tx *dbs.Tx, cacheMap *utils.C
 
 	return result, nil
 }
+
+// CheckAPINodeIsPrimary 检查当前节点是否为Primary节点
+func (this *APINodeDAO) CheckAPINodeIsPrimary(tx *dbs.Tx) (bool, error) {
+	config, err := configs.SharedAPIConfig()
+	if err != nil {
+		return false, err
+	}
+
+	isPrimary, err := this.Query(tx).
+		State(APINodeStateEnabled).
+		Attr("uniqueId", config.NodeId).
+		Attr("isPrimary", true).
+		Exist()
+	if err != nil {
+		return false, err
+	}
+
+	if isPrimary {
+		return true, nil
+	}
+
+	// 检查是否有别的Primary节点
+	count, err := this.Query(tx).
+		State(APINodeStateEnabled).
+		Attr("isOn", true).
+		Attr("isPrimary", true).
+		Count()
+	if err != nil {
+		return false, err
+	}
+
+	if count == 0 {
+		err = this.ResetPrimaryAPINode(tx)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+// CheckAPINodeIsPrimaryWithoutErr 检查当前节点是否为Primary节点，并忽略错误
+func (this *APINodeDAO) CheckAPINodeIsPrimaryWithoutErr() bool {
+	b, err := this.CheckAPINodeIsPrimary(nil)
+	return b && err == nil
+}
+
+// ResetPrimaryAPINode 重置Primary节点
+func (this *APINodeDAO) ResetPrimaryAPINode(tx *dbs.Tx) error {
+	// 当前是否有Primary节点
+	apiNode, err := this.Query(tx).
+		State(APINodeStateEnabled).
+		Attr("isOn", true).
+		Attr("isPrimary", true).
+		Find()
+	if err != nil {
+		return err
+	}
+	if apiNode == nil {
+		// 选择一个作为Primary
+		// TODO 将来需要考虑API节点离线的情况
+		apiNodeId, err := this.Query(tx).
+			State(APINodeStateEnabled).
+			Attr("isOn", true).
+			ResultPk().
+			FindInt64Col(0)
+		if err != nil {
+			return err
+		}
+		if apiNodeId > 0 {
+			err = this.Query(tx).
+				Pk(apiNodeId).
+				Set("isPrimary", true).
+				UpdateQuickly()
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// NotifyUpdate 通知变更
+func (this *APINodeDAO) NotifyUpdate(tx *dbs.Tx, apiNodeId int64) error {
+	// suppress IDE warning
+	_ = apiNodeId
+
+	err := this.ResetPrimaryAPINode(tx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// 生成唯一ID
+func (this *APINodeDAO) genUniqueId(tx *dbs.Tx) (string, error) {
+	for {
+		uniqueId := rands.HexString(32)
+		ok, err := this.Query(tx).
+			Attr("uniqueId", uniqueId).
+			Exist()
+		if err != nil {
+			return "", err
+		}
+		if ok {
+			continue
+		}
+		return uniqueId, nil
+	}
+}

internal/db/models/api_node_dao_test.go

@@ -34,6 +34,16 @@ func TestAPINodeDAO_FindAllEnabledAPIAccessIPs(t *testing.T) {
 	t.Log(NewAPINodeDAO().FindAllEnabledAPIAccessIPs(nil, cacheMap))
 }
 
+func TestAPINodeDAO_CheckAPINodeIsPrimary(t *testing.T) {
+	var dao = NewAPINodeDAO()
+	t.Log(dao.CheckAPINodeIsPrimary(nil))
+}
+
+func TestAPINodeDAO_ResetPrimaryAPINode(t *testing.T) {
+	var dao = NewAPINodeDAO()
+	t.Log(dao.ResetPrimaryAPINode(nil))
+}
+
 func BenchmarkAPINodeDAO_New(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 	for i := 0; i < b.N; i++ {

internal/db/models/api_node_model.go

@@ -23,6 +23,7 @@ type APINode struct {
 	AdminId     uint32   `field:"adminId"`     // 管理员ID
 	Weight      uint32   `field:"weight"`      // 权重
 	Status      dbs.JSON `field:"status"`      // 运行状态
+	IsPrimary   bool     `field:"isPrimary"`   // 是否为主API节点
 }
 
 type APINodeOperator struct {
@@ -45,6 +46,7 @@ type APINodeOperator struct {
 	AdminId     interface{} // 管理员ID
 	Weight      interface{} // 权重
 	Status      interface{} // 运行状态
+	IsPrimary   interface{} // 是否为主API节点
 }
 
 func NewAPINodeOperator() *APINodeOperator {

internal/db/models/authority/authority_node_dao.go

@@ -1,6 +1,7 @@
 package authority
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
@@ -188,13 +189,19 @@ func (this *AuthorityNodeDAO) GenUniqueId(tx *dbs.Tx) (string, error) {
 }
 
 // UpdateNodeStatus 更改节点状态
-func (this *AuthorityNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, statusJSON []byte) error {
+func (this *AuthorityNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, nodeStatus *nodeconfigs.NodeStatus) error {
-	if statusJSON == nil {
+	if nodeStatus == nil {
 		return nil
 	}
-	_, err := this.Query(tx).
+
+	nodeStatusJSON, err := json.Marshal(nodeStatus)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.Query(tx).
 		Pk(nodeId).
-		Set("status", string(statusJSON)).
+		Set("status", nodeStatusJSON).
 		Update()
 	return err
 }

internal/db/models/db_node_dao.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"github.com/TeaOSLab/EdgeAPI/internal/encrypt"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -49,12 +50,17 @@ func (this *DBNodeDAO) EnableDBNode(tx *dbs.Tx, id int64) error {
 }
 
 // DisableDBNode 禁用条目
-func (this *DBNodeDAO) DisableDBNode(tx *dbs.Tx, id int64) error {
+func (this *DBNodeDAO) DisableDBNode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", DBNodeStateDisabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleDatabase, nodeId)
 }
 
 // FindEnabledDBNode 查找启用中的条目

internal/db/models/db_node_initializer.go

@@ -56,6 +56,25 @@ func init() {
 	})
 }
 
+func AllAccessLogDBs() []*dbs.DB {
+	accessLogLocker.Lock()
+	defer accessLogLocker.Unlock()
+
+	var result = []*dbs.DB{}
+	for _, db := range accessLogDBMapping {
+		result = append(result, db)
+	}
+
+	if len(result) == 0 {
+		db, _ := dbs.Default()
+		if db != nil {
+			result = append(result, db)
+		}
+	}
+
+	return result
+}
+
 // 获取获取DAO
 func randomHTTPAccessLogDAO() (dao *HTTPAccessLogDAOWrapper) {
 	accessLogLocker.RLock()
@@ -237,7 +256,7 @@ func (this *DBNodeInitializer) loop() error {
 		}
 
 		if db == nil {
-			config := &dbs.DBConfig{
+			var config = &dbs.DBConfig{
 				Driver: "mysql",
 				Dsn:    dsn,
 				Prefix: "edge",
@@ -251,7 +270,7 @@ func (this *DBNodeInitializer) loop() error {
 			// 检查表是否存在
 			// httpAccessLog
 			{
-				tableDef, err := SharedHTTPAccessLogManager.FindTable(db, timeutil.Format("Ymd"), true)
+				tableDef, err := SharedHTTPAccessLogManager.FindLastTable(db, timeutil.Format("Ymd"), true)
 				if err != nil {
 					remotelogs.Error("DB_NODE", "create first table in database node failed: "+err.Error())
 

internal/db/models/dns/dns_provider_dao.go

@@ -107,22 +107,39 @@ func (this *DNSProviderDAO) UpdateDNSProvider(tx *dbs.Tx, dnsProviderId int64, n
 }
 
 // CountAllEnabledDNSProviders 计算服务商数量
-func (this *DNSProviderDAO) CountAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string) (int64, error) {
+func (this *DNSProviderDAO) CountAllEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, domain string, providerType string) (int64, error) {
 	var query = dbutils.NewQuery(tx, this, adminId, userId)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
+
+	if len(domain) > 0 {
+		query.Where("id IN (SELECT providerId FROM " + SharedDNSDomainDAO.Table + " WHERE state=1 AND name=:domain)")
+		query.Param("domain", domain)
+	}
+
+	if len(providerType) > 0 {
+		query.Attr("type", providerType)
+	}
+
 	return query.State(DNSProviderStateEnabled).
 		Count()
 }
 
 // ListEnabledDNSProviders 列出单页服务商
-func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, offset int64, size int64) (result []*DNSProvider, err error) {
+func (this *DNSProviderDAO) ListEnabledDNSProviders(tx *dbs.Tx, adminId int64, userId int64, keyword string, domain string, providerType string, offset int64, size int64) (result []*DNSProvider, err error) {
 	var query = dbutils.NewQuery(tx, this, adminId, userId)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
+	}
+	if len(domain) > 0 {
+		query.Where("id IN (SELECT providerId FROM " + SharedDNSDomainDAO.Table + " WHERE state=1 AND name=:domain)")
+		query.Param("domain", domain)
+	}
+	if len(providerType) > 0 {
+		query.Attr("type", providerType)
 	}
 	_, err = query.
 		State(DNSProviderStateEnabled).

internal/db/models/dns/dns_task_dao.go

@@ -12,10 +12,11 @@ import (
 type DNSTaskType = string
 
 const (
-	DNSTaskTypeClusterChange DNSTaskType = "clusterChange"
+	DNSTaskTypeClusterChange       DNSTaskType = "clusterChange"
-	DNSTaskTypeNodeChange    DNSTaskType = "nodeChange"
+	DNSTaskTypeClusterRemoveDomain DNSTaskType = "clusterRemoveDomain" // 从集群中移除域名
-	DNSTaskTypeServerChange  DNSTaskType = "serverChange"
+	DNSTaskTypeNodeChange          DNSTaskType = "nodeChange"
-	DNSTaskTypeDomainChange  DNSTaskType = "domainChange"
+	DNSTaskTypeServerChange        DNSTaskType = "serverChange"
+	DNSTaskTypeDomainChange        DNSTaskType = "domainChange"
 )
 
 type DNSTaskDAO dbs.DAO
@@ -40,20 +41,21 @@ func init() {
 }
 
 // CreateDNSTask 生成任务
-func (this *DNSTaskDAO) CreateDNSTask(tx *dbs.Tx, clusterId int64, serverId int64, nodeId int64, domainId int64, taskType string) error {
+func (this *DNSTaskDAO) CreateDNSTask(tx *dbs.Tx, clusterId int64, serverId int64, nodeId int64, domainId int64, recordName string, taskType string) error {
 	if clusterId <= 0 && serverId <= 0 && nodeId <= 0 && domainId <= 0 {
 		return nil
 	}
 	err := this.Query(tx).InsertOrUpdateQuickly(maps.Map{
-		"clusterId": clusterId,
+		"clusterId":  clusterId,
-		"serverId":  serverId,
+		"serverId":   serverId,
-		"nodeId":    nodeId,
+		"nodeId":     nodeId,
-		"domainId":  domainId,
+		"domainId":   domainId,
-		"updatedAt": time.Now().Unix(),
+		"recordName": recordName,
-		"type":      taskType,
+		"updatedAt":  time.Now().Unix(),
-		"isDone":    false,
+		"type":       taskType,
-		"isOk":      false,
+		"isDone":     false,
-		"error":     "",
+		"isOk":       false,
+		"error":      "",
 	}, maps.Map{
 		"updatedAt": time.Now().Unix(),
 		"isDone":    false,
@@ -63,24 +65,29 @@ func (this *DNSTaskDAO) CreateDNSTask(tx *dbs.Tx, clusterId int64, serverId int6
 	return err
 }
 
-// CreateClusterTask 生成集群任务
+// CreateClusterTask 生成集群变更任务
 func (this *DNSTaskDAO) CreateClusterTask(tx *dbs.Tx, clusterId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, clusterId, 0, 0, 0, taskType)
+	return this.CreateDNSTask(tx, clusterId, 0, 0, 0, "", taskType)
+}
+
+// CreateClusterRemoveTask 生成集群删除域名任务
+func (this *DNSTaskDAO) CreateClusterRemoveTask(tx *dbs.Tx, clusterId int64, domainId int64, recordName string) error {
+	return this.CreateDNSTask(tx, clusterId, 0, 0, domainId, recordName, DNSTaskTypeClusterRemoveDomain)
 }
 
 // CreateNodeTask 生成节点任务
 func (this *DNSTaskDAO) CreateNodeTask(tx *dbs.Tx, nodeId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, 0, 0, nodeId, 0, taskType)
+	return this.CreateDNSTask(tx, 0, 0, nodeId, 0, "", taskType)
 }
 
 // CreateServerTask 生成服务任务
-func (this *DNSTaskDAO) CreateServerTask(tx *dbs.Tx, serverId int64, taskType DNSTaskType) error {
+func (this *DNSTaskDAO) CreateServerTask(tx *dbs.Tx, clusterId int64, serverId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, 0, serverId, 0, 0, taskType)
+	return this.CreateDNSTask(tx, clusterId, serverId, 0, 0, "", taskType)
 }
 
 // CreateDomainTask 生成域名更新任务
 func (this *DNSTaskDAO) CreateDomainTask(tx *dbs.Tx, domainId int64, taskType DNSTaskType) error {
-	return this.CreateDNSTask(tx, 0, 0, 0, domainId, taskType)
+	return this.CreateDNSTask(tx, 0, 0, 0, domainId, "", taskType)
 }
 
 // FindAllDoingTasks 查找所有正在执行的任务
@@ -101,6 +108,7 @@ func (this *DNSTaskDAO) FindAllDoingOrErrorTasks(tx *dbs.Tx, nodeClusterId int64
 	}
 	_, err = query.
 		Where("(isDone=0 OR (isDone=1 AND isOk=0))").
+		Asc("updatedAt").
 		AscPk().
 		Slice(&result).
 		FindAll()

internal/db/models/dns/dns_task_model.go

@@ -1,30 +1,32 @@
 package dns
 
-// DNS更新任务
+// DNSTask DNS更新任务
 type DNSTask struct {
-	Id        uint64 `field:"id"`        // ID
+	Id         uint64 `field:"id"`         // ID
-	ClusterId uint32 `field:"clusterId"` // 集群ID
+	ClusterId  uint32 `field:"clusterId"`  // 集群ID
-	ServerId  uint32 `field:"serverId"`  // 服务ID
+	ServerId   uint32 `field:"serverId"`   // 服务ID
-	NodeId    uint32 `field:"nodeId"`    // 节点ID
+	NodeId     uint32 `field:"nodeId"`     // 节点ID
-	DomainId  uint32 `field:"domainId"`  // 域名ID
+	DomainId   uint32 `field:"domainId"`   // 域名ID
-	Type      string `field:"type"`      // 任务类型
+	RecordName string `field:"recordName"` // 记录名
-	UpdatedAt uint64 `field:"updatedAt"` // 更新时间
+	Type       string `field:"type"`       // 任务类型
-	IsDone    bool   `field:"isDone"`    // 是否已完成
+	UpdatedAt  uint64 `field:"updatedAt"`  // 更新时间
-	IsOk      bool   `field:"isOk"`      // 是否成功
+	IsDone     bool   `field:"isDone"`     // 是否已完成
-	Error     string `field:"error"`     // 错误信息
+	IsOk       bool   `field:"isOk"`       // 是否成功
+	Error      string `field:"error"`      // 错误信息
 }
 
 type DNSTaskOperator struct {
-	Id        interface{} // ID
+	Id         interface{} // ID
-	ClusterId interface{} // 集群ID
+	ClusterId  interface{} // 集群ID
-	ServerId  interface{} // 服务ID
+	ServerId   interface{} // 服务ID
-	NodeId    interface{} // 节点ID
+	NodeId     interface{} // 节点ID
-	DomainId  interface{} // 域名ID
+	DomainId   interface{} // 域名ID
-	Type      interface{} // 任务类型
+	RecordName interface{} // 记录名
-	UpdatedAt interface{} // 更新时间
+	Type       interface{} // 任务类型
-	IsDone    interface{} // 是否已完成
+	UpdatedAt  interface{} // 更新时间
-	IsOk      interface{} // 是否成功
+	IsDone     interface{} // 是否已完成
-	Error     interface{} // 错误信息
+	IsOk       interface{} // 是否成功
+	Error      interface{} // 错误信息
 }
 
 func NewDNSTaskOperator() *DNSTaskOperator {

internal/db/models/dns/dnsutils/dns_utils.go

@@ -153,7 +153,7 @@ func CheckClusterDNS(tx *dbs.Tx, cluster *models.NodeCluster) (issues []*pb.DNSI
 		}
 
 		// 检查IP地址
-		ipAddr, _, err := models.SharedNodeIPAddressDAO.FindFirstNodeAccessIPAddress(tx, nodeId, nodeconfigs.NodeRoleNode)
+		ipAddr, _, err := models.SharedNodeIPAddressDAO.FindFirstNodeAccessIPAddress(tx, nodeId, true, nodeconfigs.NodeRoleNode)
 		if err != nil {
 			return nil, err
 		}

internal/db/models/file_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -33,7 +34,7 @@ func init() {
 	})
 }
 
-// 启用条目
+// EnableFile 启用条目
 func (this *FileDAO) EnableFile(tx *dbs.Tx, id int64) error {
 	_, err := this.Query(tx).
 		Pk(id).
@@ -42,7 +43,7 @@ func (this *FileDAO) EnableFile(tx *dbs.Tx, id int64) error {
 	return err
 }
 
-// 禁用条目
+// DisableFile 禁用条目
 func (this *FileDAO) DisableFile(tx *dbs.Tx, id int64) error {
 	_, err := this.Query(tx).
 		Pk(id).
@@ -51,7 +52,7 @@ func (this *FileDAO) DisableFile(tx *dbs.Tx, id int64) error {
 	return err
 }
 
-// 查找启用中的条目
+// FindEnabledFile 查找启用中的条目
 func (this *FileDAO) FindEnabledFile(tx *dbs.Tx, id int64) (*File, error) {
 	result, err := this.Query(tx).
 		Pk(id).
@@ -63,9 +64,9 @@ func (this *FileDAO) FindEnabledFile(tx *dbs.Tx, id int64) (*File, error) {
 	return result.(*File), err
 }
 
-// 创建文件
+// CreateFile 创建文件
 func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, businessType, description string, filename string, size int64, isPublic bool) (int64, error) {
-	op := NewFileOperator()
+	var op = NewFileOperator()
 	op.AdminId = adminId
 	op.UserId = userId
 	op.Type = businessType
@@ -74,6 +75,7 @@ func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, busines
 	op.Size = size
 	op.Filename = filename
 	op.IsPublic = isPublic
+	op.Code = utils.Sha1RandomString()
 	err := this.Save(tx, op)
 	if err != nil {
 		return 0, err
@@ -82,7 +84,7 @@ func (this *FileDAO) CreateFile(tx *dbs.Tx, adminId int64, userId int64, busines
 	return types.Int64(op.Id), nil
 }
 
-// 将文件置为已完成
+// UpdateFileIsFinished 将文件置为已完成
 func (this *FileDAO) UpdateFileIsFinished(tx *dbs.Tx, fileId int64) error {
 	_, err := this.Query(tx).
 		Pk(fileId).

internal/db/models/file_model.go

@@ -1,9 +1,10 @@
 package models
 
-// 文件管理
+// File 文件管理
 type File struct {
 	Id          uint32 `field:"id"`          // ID
 	AdminId     uint32 `field:"adminId"`     // 管理员ID
+	Code        string `field:"code"`        // 代号
 	UserId      uint32 `field:"userId"`      // 用户ID
 	Description string `field:"description"` // 文件描述
 	Filename    string `field:"filename"`    // 文件名
@@ -19,6 +20,7 @@ type File struct {
 type FileOperator struct {
 	Id          interface{} // ID
 	AdminId     interface{} // 管理员ID
+	Code        interface{} // 代号
 	UserId      interface{} // 用户ID
 	Description interface{} // 文件描述
 	Filename    interface{} // 文件名

internal/db/models/http_access_log_dao.go

@@ -3,6 +3,7 @@ package models
 import (
 	"bytes"
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -22,6 +23,7 @@ import (
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"net"
 	"net/http"
+	"net/url"
 	"regexp"
 	"sort"
 	"strings"
@@ -181,7 +183,7 @@ Loop:
 // CreateHTTPAccessLog 写入单条访问日志
 func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLogDAO, accessLog *pb.HTTPAccessLog) error {
 	var day = timeutil.FormatTime("Ymd", accessLog.Timestamp)
-	tableDef, err := SharedHTTPAccessLogManager.FindTable(dao.Instance, day, true)
+	tableDef, err := SharedHTTPAccessLogManager.FindLastTable(dao.Instance, day, true)
 	if err != nil {
 		return err
 	}
@@ -248,7 +250,9 @@ func (this *HTTPAccessLogDAO) CreateHTTPAccessLog(tx *dbs.Tx, dao *HTTPAccessLog
 }
 
 // ListAccessLogs 读取往前的 单页访问日志
-func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
+func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx,
+	partition int32,
+	lastRequestId string,
 	size int64,
 	day string,
 	hourFrom string,
@@ -275,18 +279,19 @@ func (this *HTTPAccessLogDAO) ListAccessLogs(tx *dbs.Tx, lastRequestId string,
 		size = 1000
 	}
 
-	result, nextLastRequestId, err = this.listAccessLogs(tx, lastRequestId, size, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
+	result, nextLastRequestId, err = this.listAccessLogs(tx, partition, lastRequestId, size, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	if err != nil || int64(len(result)) < size {
 		return
 	}
 
-	moreResult, _, _ := this.listAccessLogs(tx, nextLastRequestId, 1, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
+	moreResult, _, _ := this.listAccessLogs(tx, partition, nextLastRequestId, 1, day, hourFrom, hourTo, clusterId, nodeId, serverId, reverse, hasError, firewallPolicyId, firewallRuleGroupId, firewallRuleSetId, hasFirewallPolicy, userId, keyword, ip, domain)
 	hasMore = len(moreResult) > 0
 	return
 }
 
 // 读取往前的单页访问日志
 func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
+	partition int32,
 	lastRequestId string,
 	size int64,
 	day string,
@@ -309,7 +314,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 		return nil, lastRequestId, nil
 	}
 
-	serverIds := []int64{}
+	var serverIds = []int64{}
 	if userId > 0 {
 		serverIds, err = SharedServerDAO.FindAllEnabledServerIdsWithUserId(tx, userId)
 		if err != nil {
@@ -321,7 +326,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 	}
 
 	accessLogLocker.RLock()
-	daoList := []*HTTPAccessLogDAOWrapper{}
+	var daoList = []*HTTPAccessLogDAOWrapper{}
 	for _, daoWrapper := range httpAccessLogDAOMapping {
 		daoList = append(daoList, daoWrapper)
 	}
@@ -339,7 +344,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 	if clusterId > 0 {
 		nodeIds, err = SharedNodeDAO.FindAllEnabledNodeIdsWithClusterId(tx, clusterId)
 		if err != nil {
-			remotelogs.Error("DBNODE", err.Error())
+			remotelogs.Error("DB_NODE", err.Error())
 			return
 		}
 		sort.Slice(nodeIds, func(i, j int) bool {
@@ -349,32 +354,61 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 
 	// 准备查询
 	var tableQueries = []*accessLogTableQuery{}
+	var maxTableName = ""
 	for _, daoWrapper := range daoList {
 		var instance = daoWrapper.DAO.Instance
-		tableDefs, err := SharedHTTPAccessLogManager.FindTables(instance, day)
+		def, err := SharedHTTPAccessLogManager.FindPartitionTable(instance, day, partition)
 		if err != nil {
 			return nil, "", err
 		}
-		for _, def := range tableDefs {
+		if !def.Exists {
-			tableQueries = append(tableQueries, &accessLogTableQuery{
+			continue
-				daoWrapper:         daoWrapper,
-				name:               def.Name,
-				hasRemoteAddrField: def.HasRemoteAddr,
-				hasDomainField:     def.HasDomain,
-			})
 		}
+
+		if len(maxTableName) == 0 || def.Name > maxTableName {
+			maxTableName = def.Name
+		}
+
+		tableQueries = append(tableQueries, &accessLogTableQuery{
+			daoWrapper:         daoWrapper,
+			name:               def.Name,
+			hasRemoteAddrField: def.HasRemoteAddr,
+			hasDomainField:     def.HasDomain,
+		})
+	}
+
+	// 检查各个分表是否一致
+	if partition < 0 {
+		var newTableQueries = []*accessLogTableQuery{}
+		for _, tableQuery := range tableQueries {
+			if tableQuery.name != maxTableName {
+				continue
+			}
+			newTableQueries = append(newTableQueries, tableQuery)
+		}
+		tableQueries = newTableQueries
+	}
+
+	if len(tableQueries) == 0 {
+		return nil, "", nil
 	}
 
 	var locker = sync.Mutex{}
 
+	// 这里正则表达式中的括号不能轻易变更，因为后面有引用
+	// TODO 支持多个查询条件的组合，比如 status:200 proto:HTTP/1.1
 	var statusPrefixReg = regexp.MustCompile(`status:\s*(\d{3})\b`)
 	var statusRangeReg = regexp.MustCompile(`status:\s*(\d{3})-(\d{3})\b`)
+	var urlReg = regexp.MustCompile(`^(http|https)://`)
+	var requestPathReg = regexp.MustCompile(`requestPath:(\S+)`)
+	var protoReg = regexp.MustCompile(`proto:(\S+)`)
+	var schemeReg = regexp.MustCompile(`scheme:(\S+)`)
 
 	var count = len(tableQueries)
 	var wg = &sync.WaitGroup{}
 	wg.Add(count)
 	for _, tableQuery := range tableQueries {
-		go func(tableQuery *accessLogTableQuery) {
+		go func(tableQuery *accessLogTableQuery, keyword string) {
 			defer wg.Done()
 
 			var dao = tableQuery.daoWrapper.DAO
@@ -462,27 +496,53 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 			}
 
 			if len(keyword) > 0 {
-				// remoteAddr
+				var isSpecialKeyword = false
-				if tableQuery.hasRemoteAddrField && net.ParseIP(keyword) != nil {
+
+				if tableQuery.hasRemoteAddrField && net.ParseIP(keyword) != nil { // ip
+					isSpecialKeyword = true
 					query.Attr("remoteAddr", keyword)
-				} else if tableQuery.hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
+				} else if tableQuery.hasRemoteAddrField && regexp.MustCompile(`^ip:.+`).MatchString(keyword) { // ip:x.x.x.x
+					isSpecialKeyword = true
 					keyword = keyword[3:]
 					pieces := strings.SplitN(keyword, ",", 2)
-					if len(pieces) == 1 || len(pieces[1]) == 0 {
+					if len(pieces) == 1 || len(pieces[1]) == 0 || pieces[0] == pieces[1] {
 						query.Attr("remoteAddr", pieces[0])
 					} else {
 						query.Between("INET_ATON(remoteAddr)", utils.IP2Long(pieces[0]), utils.IP2Long(pieces[1]))
 					}
-				} else if statusRangeReg.MatchString(keyword) {
+				} else if statusRangeReg.MatchString(keyword) { // status:200-400
+					isSpecialKeyword = true
 					var matches = statusRangeReg.FindStringSubmatch(keyword)
 					query.Between("status", types.Int(matches[1]), types.Int(matches[2]))
-
+				} else if statusPrefixReg.MatchString(keyword) { // status:200
-					// TODO 处理剩余的关键词
+					isSpecialKeyword = true
-				} else if statusPrefixReg.MatchString(keyword) {
 					var matches = statusPrefixReg.FindStringSubmatch(keyword)
 					query.Attr("status", matches[1])
-					// TODO 处理剩余的关键词
+				} else if requestPathReg.MatchString(keyword) {
-				} else {
+					isSpecialKeyword = true
+					var matches = requestPathReg.FindStringSubmatch(keyword)
+					query.Where("JSON_EXTRACT(content, '$.requestPath')=:keyword").
+						Param("keyword", matches[1])
+				} else if protoReg.MatchString(keyword) {
+					isSpecialKeyword = true
+					var matches = protoReg.FindStringSubmatch(keyword)
+					query.Where("JSON_EXTRACT(content, '$.proto')=:keyword").
+						Param("keyword", strings.ToUpper(matches[1]))
+				} else if schemeReg.MatchString(keyword) {
+					isSpecialKeyword = true
+					var matches = schemeReg.FindStringSubmatch(keyword)
+					query.Where("JSON_EXTRACT(content, '$.scheme')=:keyword").
+						Param("keyword", strings.ToLower(matches[1]))
+				} else if urlReg.MatchString(keyword) { // https://xxx/yyy
+					u, err := url.Parse(keyword)
+					if err == nil {
+						isSpecialKeyword = true
+						query.Attr("domain", u.Host)
+						query.Where("JSON_EXTRACT(content, '$.requestURI') LIKE :keyword").
+							Param("keyword", dbutils.QuoteLikePrefix("\""+u.RequestURI()))
+					}
+				}
+				if !isSpecialKeyword {
 					if regexp.MustCompile(`^ip:.+`).MatchString(keyword) {
 						keyword = keyword[3:]
 					}
@@ -530,7 +590,7 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 					}
 
 					query.Where("("+where+")").
-						Param("keyword", "%"+keyword+"%")
+						Param("keyword", dbutils.QuoteLike(keyword))
 					if useOriginKeyword {
 						query.Param("originKeyword", keyword)
 					}
@@ -574,17 +634,17 @@ func (this *HTTPAccessLogDAO) listAccessLogs(tx *dbs.Tx,
 				Limit(size).
 				FindAll()
 			if err != nil {
-				logs.Println("[DB_NODE]" + err.Error())
+				remotelogs.Println("DB_NODE", err.Error())
 				return
 			}
 
 			locker.Lock()
 			for _, one := range ones {
-				accessLog := one.(*HTTPAccessLog)
+				var accessLog = one.(*HTTPAccessLog)
 				result = append(result, accessLog)
 			}
 			locker.Unlock()
-		}(tableQuery)
+		}(tableQuery, keyword)
 	}
 	wg.Wait()
 

internal/db/models/http_access_log_dao_test.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
@@ -53,7 +54,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "", 10, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, "", 10, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -80,7 +81,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page(t *testing.T) {
 	times := 0 // 防止循环次数太多
 	for {
 		before := time.Now()
-		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, lastRequestId, 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 		cost := time.Since(before).Seconds()
 		if err != nil {
 			t.Fatal(err)
@@ -111,7 +112,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Reverse(t *testing.T) {
 	}
 
 	before := time.Now()
-	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
+	accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, "16023261176446590001000000000000003500000004", 2, timeutil.Format("Ymd"), "", "", 0, 0, 0, true, false, 0, 0, 0, false, 0, "", "", "")
 	cost := time.Since(before).Seconds()
 	if err != nil {
 		t.Fatal(err)
@@ -136,7 +137,7 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
 	times := 0 // 防止循环次数太多
 	for {
 		before := time.Now()
-		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
+		accessLogs, requestId, hasMore, err := SharedHTTPAccessLogDAO.ListAccessLogs(tx, -1, lastRequestId, 2, timeutil.Format("Ymd", time.Now().AddDate(0, 0, 1)), "", "", 0, 0, 0, false, false, 0, 0, 0, false, 0, "", "", "")
 		cost := time.Since(before).Seconds()
 		if err != nil {
 			t.Fatal(err)
@@ -157,3 +158,13 @@ func TestHTTPAccessLogDAO_ListAccessLogs_Page_NotExists(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkHTTPAccessLogDAO_JSONEncode(b *testing.B) {
+	var accessLog = &pb.HTTPAccessLog{
+		RequestPath: "/hello/world",
+	}
+
+	for i := 0; i < b.N; i++ {
+		_, _ = json.Marshal(accessLog)
+	}
+}

internal/db/models/http_access_log_manager.go

@@ -15,6 +15,7 @@ import (
 )
 
 // 访问日志的两个表格形式
+// 括号位置需要固定，会用来读取日期和分区
 var accessLogTableMainReg = regexp.MustCompile(`_(\d{8})$`)
 var accessLogTablePartialReg = regexp.MustCompile(`_(\d{8})_(\d{4})$`)
 
@@ -38,7 +39,7 @@ func (this *HTTPAccessLogManager) FindTableNames(db *dbs.DB, day string) ([]stri
 
 	// 需要防止用户设置了表名自动小写
 	for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
-		ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
+		ones, columnNames, err := db.FindPreparedOnes(`SHOW TABLES LIKE '` + prefix + `'`)
 		if err != nil {
 			return nil, errors.New("query table names error: " + err.Error())
 		}
@@ -77,9 +78,15 @@ func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAcc
 	var results = []*httpAccessLogDefinition{}
 	var tableNames = []string{}
 
+	config, err := db.Config()
+	if err != nil {
+		return nil, err
+	}
+	var cachePrefix = config.Dsn
+
 	// 需要防止用户设置了表名自动小写
 	for _, prefix := range []string{"edgeHTTPAccessLogs_" + day + "%", "edgehttpaccesslogs_" + day + "%"} {
-		ones, columnNames, err := db.FindOnes(`SHOW TABLES LIKE '` + prefix + `'`)
+		ones, columnNames, err := db.FindPreparedOnes(`SHOW TABLES LIKE '` + prefix + `'`)
 		if err != nil {
 			return nil, errors.New("query table names error: " + err.Error())
 		}
@@ -96,17 +103,32 @@ func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAcc
 			if accessLogTableMainReg.MatchString(tableName) {
 				tableNames = append(tableNames, tableName)
 
-				hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, tableName)
+				// 查找已有的表格信息，避免SHOW FIELDS
-				if err != nil {
+				var tableDay = tableName[strings.LastIndex(tableName, "_")+1:]
-					return nil, err
+				var cacheKey = this.composeTableCacheKey(cachePrefix, tableDay)
-				}
+				this.locker.Lock()
+				currentTableDef, ok := this.currentTableMapping[cacheKey]
+				this.locker.Unlock()
+				if ok {
+					results = append(results, &httpAccessLogDefinition{
+						Name:          tableName,
+						HasRemoteAddr: currentTableDef.HasRemoteAddr,
+						HasDomain:     currentTableDef.HasDomain,
+						Exists:        true,
+					})
+				} else {
+					hasRemoteAddrField, hasDomainField, err := this.checkTableFields(db, tableName)
+					if err != nil {
+						return nil, err
+					}
 
-				results = append(results, &httpAccessLogDefinition{
+					results = append(results, &httpAccessLogDefinition{
-					Name:          tableName,
+						Name:          tableName,
-					HasRemoteAddr: hasRemoteAddrField,
+						HasRemoteAddr: hasRemoteAddrField,
-					HasDomain:     hasDomainField,
+						HasDomain:     hasDomainField,
-					Exists:        true,
+						Exists:        true,
-				})
+					})
+				}
 			} else if accessLogTablePartialReg.MatchString(tableName) {
 				tableNames = append(tableNames, tableName)
 
@@ -128,11 +150,55 @@ func (this *HTTPAccessLogManager) FindTables(db *dbs.DB, day string) ([]*httpAcc
 	return results, nil
 }
 
-// FindTable 根据日期获取表名
+func (this *HTTPAccessLogManager) FindPartitionTable(db *dbs.DB, day string, partition int32) (*httpAccessLogDefinition, error) {
+	var tableNames []string
+	if partition < 0 {
+		tableList, err := this.FindTables(db, day)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(tableList) > 0 {
+			return tableList[len(tableList)-1], nil
+		}
+
+		return &httpAccessLogDefinition{
+			Name:          "",
+			HasRemoteAddr: false,
+			HasDomain:     false,
+			Exists:        false,
+		}, nil
+	} else if partition == 0 {
+		tableNames = []string{"edgeHTTPAccessLogs_" + day, "edgehttpaccesslogs_" + day}
+	} else {
+		tableNames = []string{"edgeHTTPAccessLogs_" + day + "_" + fmt.Sprintf("%04d", partition), "edgehttpaccesslogs_" + day + "_" + fmt.Sprintf("%04d", partition)}
+	}
+	for _, tableName := range tableNames {
+		hasRemoteField, hasDomainField, err := this.checkTableFields(db, tableName)
+		if err != nil {
+			continue
+		}
+		return &httpAccessLogDefinition{
+			Name:          tableName,
+			HasRemoteAddr: hasRemoteField,
+			HasDomain:     hasDomainField,
+			Exists:        true,
+		}, nil
+	}
+
+	return &httpAccessLogDefinition{
+		Name:          "",
+		HasRemoteAddr: false,
+		HasDomain:     false,
+		Exists:        false,
+	}, nil
+}
+
+// FindLastTable 根据日期获取上一个可以使用的表名
 // 表名组成
 //   - PREFIX_DAY
 //   - PREFIX_DAY_0001
-func (this *HTTPAccessLogManager) FindTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
+func (this *HTTPAccessLogManager) FindLastTable(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
 	this.locker.Lock()
 	defer this.locker.Unlock()
 
@@ -197,6 +263,60 @@ func (this *HTTPAccessLogManager) ResetTable(db *dbs.DB, day string) {
 	delete(this.currentTableMapping, this.composeTableCacheKey(config.Dsn, day))
 }
 
+// TablePartition 从表名中获取分区
+func (this *HTTPAccessLogManager) TablePartition(tableName string) (partition int32) {
+	if accessLogTablePartialReg.MatchString(tableName) {
+		return types.Int32(accessLogTablePartialReg.FindStringSubmatch(tableName)[2])
+	}
+
+	return 0
+}
+
+// FindLatestPartition 读取最后一个分区
+func (this *HTTPAccessLogManager) FindLatestPartition(day string) (int32, error) {
+	var dbList = AllAccessLogDBs()
+	if len(dbList) == 0 {
+		return 0, errors.New("no valid database")
+	}
+
+	var partitions = []int32{}
+	var locker sync.Mutex
+
+	var wg = sync.WaitGroup{}
+	wg.Add(len(dbList))
+
+	var lastErr error
+	for _, db := range dbList {
+		go func(db *dbs.DB) {
+			defer wg.Done()
+
+			names, err := this.FindTableNames(db, day)
+			if err != nil {
+				lastErr = err
+			}
+			for _, name := range names {
+				var partition = this.TablePartition(name)
+				locker.Lock()
+				if !lists.Contains(partitions, partition) {
+					partitions = append(partitions, partition)
+				}
+				locker.Unlock()
+			}
+		}(db)
+	}
+	wg.Wait()
+
+	if lastErr != nil {
+		return 0, lastErr
+	}
+
+	if len(partitions) == 0 {
+		return 0, nil
+	}
+
+	return partitions[len(partitions)-1], nil
+}
+
 // 查找某个表格
 func (this *HTTPAccessLogManager) findTableWithoutCache(db *dbs.DB, day string, force bool) (*httpAccessLogDefinition, error) {
 	tableNames, err := this.FindTableNames(db, day)
@@ -296,7 +416,7 @@ func (this *HTTPAccessLogManager) findTableWithoutCache(db *dbs.DB, day string,
 
 // TODO 考虑缓存检查结果
 func (this *HTTPAccessLogManager) checkTableFields(db *dbs.DB, tableName string) (hasRemoteAddrField bool, hasDomainField bool, err error) {
-	fields, _, err := db.FindOnes("SHOW FIELDS FROM " + tableName)
+	fields, _, err := db.FindPreparedOnes("SHOW FIELDS FROM " + tableName)
 	if err != nil {
 		return false, false, err
 	}

internal/db/models/http_access_log_manager_test.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"testing"
 	"time"
 )
@@ -30,6 +31,9 @@ func TestNewHTTPAccessLogManager(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	var manager = models.SharedHTTPAccessLogManager
 	err = manager.CreateTable(db, "accessLog_1")
@@ -58,6 +62,9 @@ func TestHTTPAccessLogManager_FindTableNames(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	for i := 0; i < 3; i++ {
 		var before = time.Now()
@@ -74,7 +81,6 @@ func TestHTTPAccessLogManager_FindTableNames(t *testing.T) {
 	}
 }
 
-
 func TestHTTPAccessLogManager_FindTables(t *testing.T) {
 	var config = &dbs.DBConfig{
 		Driver: "mysql",
@@ -95,6 +101,9 @@ func TestHTTPAccessLogManager_FindTables(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	for i := 0; i < 3; i++ {
 		var before = time.Now()
@@ -111,7 +120,7 @@ func TestHTTPAccessLogManager_FindTables(t *testing.T) {
 	}
 }
 
-func TestHTTPAccessLogManager_FindTable(t *testing.T) {
+func TestHTTPAccessLogManager_FindLastTable(t *testing.T) {
 	var config = &dbs.DBConfig{
 		Driver: "mysql",
 		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
@@ -131,10 +140,13 @@ func TestHTTPAccessLogManager_FindTable(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() {
+		_ = db.Close()
+	}()
 
 	for i := 0; i < 3; i++ {
 		var before = time.Now()
-		tableDef, err := models.SharedHTTPAccessLogManager.FindTable(db, "20220306", false)
+		tableDef, err := models.SharedHTTPAccessLogManager.FindLastTable(db, "20220306", false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -146,3 +158,32 @@ func TestHTTPAccessLogManager_FindTable(t *testing.T) {
 		t.Log(time.Since(before).Seconds()*1000, "ms")
 	}
 }
+
+func TestHTTPAccessLogManager_FindPartitionTable(t *testing.T) {
+	var config = &dbs.DBConfig{
+		Driver: "mysql",
+		Dsn:    "root:123456@tcp(127.0.0.1:3306)/db_edge_log?charset=utf8mb4&timeout=30s",
+		Prefix: "edge",
+		Connections: struct {
+			Pool         int           `yaml:"pool"`
+			Max          int           `yaml:"max"`
+			Life         string        `yaml:"life"`
+			LifeDuration time.Duration `yaml:",omitempty"`
+		}{},
+		Models: struct {
+			Package string `yaml:"package"`
+		}{},
+	}
+
+	db, err := dbs.NewInstanceFromConfig(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		_ = db.Close()
+	}()
+
+	t.Log(models.SharedHTTPAccessLogManager.FindPartitionTable(db, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), -1))
+	t.Log(models.SharedHTTPAccessLogManager.FindPartitionTable(db, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), 0))
+	t.Log(models.SharedHTTPAccessLogManager.FindPartitionTable(db, timeutil.Format("Ymd", time.Now().AddDate(0, 0, -1)), 1))
+}

internal/db/models/http_access_log_policy_dao.go

@@ -1,13 +1,10 @@
 package models
 
 import (
-	"bytes"
-	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
-	"github.com/iwind/TeaGo/maps"
 )
 
 const (
@@ -109,7 +106,7 @@ func (this *HTTPAccessLogPolicyDAO) FindAllEnabledAndOnPolicies(tx *dbs.Tx) (res
 }
 
 // CreatePolicy 创建策略
-func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool) (policyId int64, err error) {
+func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policyType string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool) (policyId int64, err error) {
 	var op = NewHTTPAccessLogPolicyOperator()
 	op.Name = name
 	op.Type = policyType
@@ -121,12 +118,13 @@ func (this *HTTPAccessLogPolicyDAO) CreatePolicy(tx *dbs.Tx, name string, policy
 	}
 	op.IsPublic = isPublic
 	op.IsOn = true
+	op.FirewallOnly = firewallOnly
 	op.State = HTTPAccessLogPolicyStateEnabled
 	return this.SaveInt64(tx, op)
 }
 
 // UpdatePolicy 修改策略
-func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, isOn bool) error {
+func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, name string, optionsJSON []byte, condsJSON []byte, isPublic bool, firewallOnly bool, isOn bool) error {
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
@@ -140,7 +138,6 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
 	if oldOne == nil {
 		return nil
 	}
-	var oldPolicy = oldOne.(*HTTPAccessLogPolicy)
 
 	var op = NewHTTPAccessLogPolicyOperator()
 	op.Id = policyId
@@ -156,22 +153,11 @@ func (this *HTTPAccessLogPolicyDAO) UpdatePolicy(tx *dbs.Tx, policyId int64, nam
 		op.Conds = "{}"
 	}
 
-	// 版本号
+	// 版本号总是加1
-	if len(oldPolicy.Options) == 0 || len(optionsJSON) == 0 {
+	op.Version = dbs.SQL("version+1")
-		op.Version = dbs.SQL("version+1")
-	} else {
-		var m1 = maps.Map{}
-		_ = json.Unmarshal(oldPolicy.Options, &m1)
-
-		var m2 = maps.Map{}
-		_ = json.Unmarshal(optionsJSON, &m2)
-
-		if bytes.Compare(m1.AsJSON(), m2.AsJSON()) != 0 {
-			op.Version = dbs.SQL("version+1")
-		}
-	}
 
 	op.IsPublic = isPublic
+	op.FirewallOnly = firewallOnly
 	op.IsOn = isOn
 	return this.Save(tx, op)
 }

internal/db/models/http_access_log_policy_model.go

@@ -4,35 +4,37 @@ import "github.com/iwind/TeaGo/dbs"
 
 // HTTPAccessLogPolicy 访问日志策略
 type HTTPAccessLogPolicy struct {
-	Id         uint32   `field:"id"`         // ID
+	Id           uint32   `field:"id"`           // ID
-	TemplateId uint32   `field:"templateId"` // 模版ID
+	TemplateId   uint32   `field:"templateId"`   // 模版ID
-	AdminId    uint32   `field:"adminId"`    // 管理员ID
+	AdminId      uint32   `field:"adminId"`      // 管理员ID
-	UserId     uint32   `field:"userId"`     // 用户ID
+	UserId       uint32   `field:"userId"`       // 用户ID
-	State      uint8    `field:"state"`      // 状态
+	State        uint8    `field:"state"`        // 状态
-	CreatedAt  uint64   `field:"createdAt"`  // 创建时间
+	CreatedAt    uint64   `field:"createdAt"`    // 创建时间
-	Name       string   `field:"name"`       // 名称
+	Name         string   `field:"name"`         // 名称
-	IsOn       bool     `field:"isOn"`       // 是否启用
+	IsOn         bool     `field:"isOn"`         // 是否启用
-	Type       string   `field:"type"`       // 存储类型
+	Type         string   `field:"type"`         // 存储类型
-	Options    dbs.JSON `field:"options"`    // 存储选项
+	Options      dbs.JSON `field:"options"`      // 存储选项
-	Conds      dbs.JSON `field:"conds"`      // 请求条件
+	Conds        dbs.JSON `field:"conds"`        // 请求条件
-	IsPublic   bool     `field:"isPublic"`   // 是否为公用
+	IsPublic     bool     `field:"isPublic"`     // 是否为公用
-	Version    uint32   `field:"version"`    // 版本号
+	FirewallOnly uint8    `field:"firewallOnly"` // 是否只记录防火墙相关
+	Version      uint32   `field:"version"`      // 版本号
 }
 
 type HTTPAccessLogPolicyOperator struct {
-	Id         interface{} // ID
+	Id           interface{} // ID
-	TemplateId interface{} // 模版ID
+	TemplateId   interface{} // 模版ID
-	AdminId    interface{} // 管理员ID
+	AdminId      interface{} // 管理员ID
-	UserId     interface{} // 用户ID
+	UserId       interface{} // 用户ID
-	State      interface{} // 状态
+	State        interface{} // 状态
-	CreatedAt  interface{} // 创建时间
+	CreatedAt    interface{} // 创建时间
-	Name       interface{} // 名称
+	Name         interface{} // 名称
-	IsOn       interface{} // 是否启用
+	IsOn         interface{} // 是否启用
-	Type       interface{} // 存储类型
+	Type         interface{} // 存储类型
-	Options    interface{} // 存储选项
+	Options      interface{} // 存储选项
-	Conds      interface{} // 请求条件
+	Conds        interface{} // 请求条件
-	IsPublic   interface{} // 是否为公用
+	IsPublic     interface{} // 是否为公用
-	Version    interface{} // 版本号
+	FirewallOnly interface{} // 是否只记录防火墙相关
+	Version      interface{} // 版本号
 }
 
 func NewHTTPAccessLogPolicyOperator() *HTTPAccessLogPolicyOperator {

internal/db/models/http_cache_policy_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
@@ -318,7 +319,7 @@ func (this *HTTPCachePolicyDAO) CountAllEnabledHTTPCachePolicies(tx *dbs.Tx, clu
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(storageType) > 0 {
 		query.Attr("type", storageType)
@@ -336,7 +337,7 @@ func (this *HTTPCachePolicyDAO) ListEnabledHTTPCachePolicies(tx *dbs.Tx, cluster
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(storageType) > 0 {
 		query.Attr("type", storageType)

internal/db/models/http_cache_task_dao.go

@@ -0,0 +1,254 @@
+package models
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+const (
+	HTTPCacheTaskStateEnabled  = 1 // 已启用
+	HTTPCacheTaskStateDisabled = 0 // 已禁用
+)
+
+type HTTPCacheTaskType = string
+
+const (
+	HTTPCacheTaskTypePurge HTTPCacheTaskType = "purge"
+	HTTPCacheTaskTypeFetch HTTPCacheTaskType = "fetch"
+)
+
+type HTTPCacheTaskDAO dbs.DAO
+
+func init() {
+	dbs.OnReadyDone(func() {
+		// 清理数据任务
+		var ticker = time.NewTicker(time.Duration(rands.Int(24, 48)) * time.Hour)
+		goman.New(func() {
+			for range ticker.C {
+				err := SharedHTTPCacheTaskDAO.Clean(nil, 30) // 只保留N天
+				if err != nil {
+					remotelogs.Error("HTTPCacheTaskDAO", "clean expired data failed: "+err.Error())
+				}
+			}
+		})
+	})
+}
+
+func NewHTTPCacheTaskDAO() *HTTPCacheTaskDAO {
+	return dbs.NewDAO(&HTTPCacheTaskDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeHTTPCacheTasks",
+			Model:  new(HTTPCacheTask),
+			PkName: "id",
+		},
+	}).(*HTTPCacheTaskDAO)
+}
+
+var SharedHTTPCacheTaskDAO *HTTPCacheTaskDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedHTTPCacheTaskDAO = NewHTTPCacheTaskDAO()
+	})
+}
+
+// EnableHTTPCacheTask 启用条目
+func (this *HTTPCacheTaskDAO) EnableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
+	_, err := this.Query(tx).
+		Pk(taskId).
+		Set("state", HTTPCacheTaskStateEnabled).
+		Update()
+	return err
+}
+
+// DisableHTTPCacheTask 禁用条目
+func (this *HTTPCacheTaskDAO) DisableHTTPCacheTask(tx *dbs.Tx, taskId int64) error {
+	_, err := this.Query(tx).
+		Pk(taskId).
+		Set("state", HTTPCacheTaskStateDisabled).
+		Update()
+	if err != nil {
+		return err
+	}
+	return this.NotifyChange(tx, taskId)
+}
+
+// FindEnabledHTTPCacheTask 查找启用中的条目
+func (this *HTTPCacheTaskDAO) FindEnabledHTTPCacheTask(tx *dbs.Tx, taskId int64) (*HTTPCacheTask, error) {
+	result, err := this.Query(tx).
+		Pk(taskId).
+		Attr("state", HTTPCacheTaskStateEnabled).
+		Find()
+	if result == nil {
+		return nil, err
+	}
+	return result.(*HTTPCacheTask), err
+}
+
+// CreateTask 创建任务
+func (this *HTTPCacheTaskDAO) CreateTask(tx *dbs.Tx, userId int64, taskType HTTPCacheTaskType, keyType string, description string) (int64, error) {
+	var op = NewHTTPCacheTaskOperator()
+	op.UserId = userId
+	op.Type = taskType
+	op.KeyType = keyType
+	op.IsOk = false
+	op.IsDone = false
+	op.IsReady = false
+	op.Description = description
+	op.Day = timeutil.Format("Ymd")
+	op.State = HTTPCacheTaskStateEnabled
+	taskId, err := this.SaveInt64(tx, op)
+	if err != nil {
+		return 0, err
+	}
+
+	err = this.NotifyChange(tx, taskId)
+	if err != nil {
+		return 0, err
+	}
+	return taskId, nil
+}
+
+// ResetTask 重置服务状态
+func (this *HTTPCacheTaskDAO) ResetTask(tx *dbs.Tx, taskId int64) error {
+	if taskId <= 0 {
+		return errors.New("invalid 'taskId'")
+	}
+
+	var op = NewHTTPCacheTaskOperator()
+	op.Id = taskId
+	op.IsOk = false
+	op.IsDone = false
+	op.DoneAt = 0
+	return this.Save(tx, op)
+}
+
+// UpdateTaskReady 设置任务为已准备
+func (this *HTTPCacheTaskDAO) UpdateTaskReady(tx *dbs.Tx, taskId int64) error {
+	return this.Query(tx).
+		Pk(taskId).
+		Set("isReady", true).
+		UpdateQuickly()
+}
+
+// CountTasks 查询所有任务数量
+func (this *HTTPCacheTaskDAO) CountTasks(tx *dbs.Tx, userId int64) (int64, error) {
+	var query = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isReady", true)
+	if userId > 0 {
+		query.Attr("userId", userId)
+	}
+	return query.Count()
+}
+
+// CountDoingTasks 查询正在执行的任务数量
+func (this *HTTPCacheTaskDAO) CountDoingTasks(tx *dbs.Tx, userId int64) (int64, error) {
+	var query = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isReady", true).
+		Attr("isDone", false)
+	if userId > 0 {
+		query.Attr("userId", userId)
+	}
+
+	return query.Count()
+}
+
+// ListTasks 列出单页任务
+func (this *HTTPCacheTaskDAO) ListTasks(tx *dbs.Tx, userId int64, offset int64, size int64) (result []*HTTPCacheTask, err error) {
+	var query = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isReady", true)
+	if userId > 0 {
+		query.Attr("userId", userId)
+	}
+	_, err = query.
+		Offset(offset).
+		Limit(size).
+		Slice(&result).
+		DescPk().
+		FindAll()
+	return
+}
+
+// ListDoingTasks 列出需要执行的任务
+func (this *HTTPCacheTaskDAO) ListDoingTasks(tx *dbs.Tx, size int64) (result []*HTTPCacheTask, err error) {
+	_, err = this.Query(tx).
+		State(HTTPCacheTaskStateEnabled).
+		Attr("isDone", false).
+		Attr("isReady", true).
+		Limit(size).
+		AscPk(). // 按照先创建先执行的原则
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// UpdateTaskStatus 标记任务已完成
+func (this *HTTPCacheTaskDAO) UpdateTaskStatus(tx *dbs.Tx, taskId int64, isDone bool, isOk bool) error {
+	if taskId <= 0 {
+		return errors.New("invalid taskId '" + types.String(taskId) + "'")
+	}
+	var op = NewHTTPCacheTaskOperator()
+	op.Id = taskId
+	op.IsDone = isDone
+	op.IsOk = isOk
+
+	if isDone {
+		op.DoneAt = time.Now().Unix()
+	}
+
+	return this.Save(tx, op)
+}
+
+// CheckUserTask 检查用户任务
+func (this *HTTPCacheTaskDAO) CheckUserTask(tx *dbs.Tx, userId int64, taskId int64) error {
+	b, err := this.Query(tx).
+		Pk(taskId).
+		Attr("userId", userId).
+		State(HTTPCacheTaskStateEnabled).
+		Exist()
+	if err != nil {
+		return err
+	}
+	if !b {
+		return ErrNotFound
+	}
+	return nil
+}
+
+// Clean 清理以往的任务
+func (this *HTTPCacheTaskDAO) Clean(tx *dbs.Tx, days int) error {
+	if days <= 0 {
+		days = 30
+	}
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
+
+	// 删除Key
+	err := SharedHTTPCacheTaskKeyDAO.Clean(tx, days)
+	if err != nil {
+		return err
+	}
+
+	// 删除任务
+	_, err = this.Query(tx).
+		Lte("day", day).
+		Delete()
+	return err
+}
+
+// NotifyChange 发送通知
+func (this *HTTPCacheTaskDAO) NotifyChange(tx *dbs.Tx, taskId int64) error {
+	// TODO
+	return nil
+}

internal/db/models/http_cache_task_dao_test.go

@@ -0,0 +1,19 @@
+package models_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	"testing"
+)
+
+func TestHTTPCacheTaskDAO_Clean(t *testing.T) {
+	dbs.NotifyReady()
+
+	err := models.SharedHTTPCacheTaskDAO.Clean(nil, 30)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

internal/db/models/http_cache_task_key_dao.go

@@ -0,0 +1,218 @@
+package models
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/errors"
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/iwind/TeaGo/Tea"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"time"
+)
+
+type HTTPCacheTaskKeyDAO dbs.DAO
+
+func NewHTTPCacheTaskKeyDAO() *HTTPCacheTaskKeyDAO {
+	return dbs.NewDAO(&HTTPCacheTaskKeyDAO{
+		DAOObject: dbs.DAOObject{
+			DB:     Tea.Env,
+			Table:  "edgeHTTPCacheTaskKeys",
+			Model:  new(HTTPCacheTaskKey),
+			PkName: "id",
+		},
+	}).(*HTTPCacheTaskKeyDAO)
+}
+
+var SharedHTTPCacheTaskKeyDAO *HTTPCacheTaskKeyDAO
+
+func init() {
+	dbs.OnReady(func() {
+		SharedHTTPCacheTaskKeyDAO = NewHTTPCacheTaskKeyDAO()
+	})
+}
+
+// CreateKey 创建Key
+// 参数：
+//   - clusterId 集群ID
+//   - nodeMapJSON 集群下节点映射，格式类似于 `{ "节点1":true, ... }`
+func (this *HTTPCacheTaskKeyDAO) CreateKey(tx *dbs.Tx, taskId int64, key string, taskType HTTPCacheTaskType, keyType string, clusterId int64) (int64, error) {
+	var op = NewHTTPCacheTaskKeyOperator()
+	op.TaskId = taskId
+	op.Key = key
+	op.Type = taskType
+	op.KeyType = keyType
+	op.ClusterId = clusterId
+
+	op.Nodes = "{}"
+	op.Errors = "{}"
+
+	return this.SaveInt64(tx, op)
+}
+
+// UpdateKeyStatus 修改Key状态
+func (this *HTTPCacheTaskKeyDAO) UpdateKeyStatus(tx *dbs.Tx, keyId int64, nodeId int64, errString string, nodesJSON []byte) error {
+	if keyId <= 0 {
+		return errors.New("invalid 'keyId'")
+	}
+
+	if len(nodesJSON) == 0 {
+		nodesJSON = []byte("{}")
+	}
+
+	taskId, err := this.Query(tx).
+		Pk(keyId).
+		Result("taskId").
+		FindInt64Col(0)
+	if err != nil {
+		return err
+	}
+
+	var jsonPath = "$.\"" + types.String(nodeId) + "\""
+
+	var query = this.Query(tx).
+		Pk(keyId).
+		Set("nodes", dbs.SQL("JSON_SET(nodes, :jsonPath1, true)")).
+		Param("jsonPath1", jsonPath)
+
+	if len(errString) > 0 {
+		query.Set("errors", dbs.SQL("JSON_SET(errors, :jsonPath2, :jsonValue2)")).
+			Param("jsonPath2", jsonPath).
+			Param("jsonValue2", errString)
+	} else {
+		query.Set("errors", dbs.SQL("JSON_REMOVE(errors, :jsonPath2)")).
+			Param("jsonPath2", jsonPath)
+	}
+
+	err = query.
+		UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	// 检查是否已完成
+	isDone, err := this.Query(tx).
+		Pk(keyId).
+		Where("JSON_CONTAINS(nodes, :nodesJSON)").
+		Param("nodesJSON", nodesJSON).
+		Exist()
+	if err != nil {
+		return err
+	}
+
+	if isDone {
+		err = this.Query(tx).
+			Pk(keyId).
+			Set("isDone", isDone).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+
+		// 检查任务是否已经完成
+		taskIsNotDone, err := this.Query(tx).
+			Attr("taskId", taskId).
+			Attr("isDone", false).
+			Exist()
+		if err != nil {
+			return err
+		}
+		var taskIsDone = !taskIsNotDone
+		var hasErrors = true
+		if taskIsDone {
+			// 已经完成，是否有错误
+			hasErrors, err = this.Query(tx).
+				Attr("taskId", taskId).
+				Where("JSON_LENGTH(errors)>0").
+				Exist()
+			if err != nil {
+				return err
+			}
+		}
+		err = SharedHTTPCacheTaskDAO.UpdateTaskStatus(tx, taskId, taskIsDone, !hasErrors)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// FindAllTaskKeys 查询某个任务下的所有Key
+func (this *HTTPCacheTaskKeyDAO) FindAllTaskKeys(tx *dbs.Tx, taskId int64) (result []*HTTPCacheTaskKey, err error) {
+	_, err = this.Query(tx).
+		Attr("taskId", taskId).
+		AscPk().
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// FindDoingTaskKeys 查询要执行的任务
+func (this *HTTPCacheTaskKeyDAO) FindDoingTaskKeys(tx *dbs.Tx, nodeId int64, size int64) (result []*HTTPCacheTaskKey, err error) {
+	// 集群ID
+	clusterIds, err := SharedNodeDAO.FindEnabledAndOnNodeClusterIds(tx, nodeId)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(clusterIds) == 0 {
+		return nil, nil
+	}
+
+	_, err = this.Query(tx).
+		Attr("clusterId", clusterIds).
+		Attr("isDone", false).
+		Where("NOT JSON_CONTAINS_PATH(nodes, 'one', :jsonPath1)").
+		Param("jsonPath1", "$.\""+types.String(nodeId)+"\"").
+		Where("taskId IN (SELECT id FROM " + SharedHTTPCacheTaskDAO.Table + " WHERE state=1 AND isReady=1 AND isDone=0)").
+		Limit(size).
+		AscPk().
+		Reuse(false).
+		Slice(&result).
+		FindAll()
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}
+
+// ResetCacheKeysWithTaskId 重置任务下的Key状态
+func (this *HTTPCacheTaskKeyDAO) ResetCacheKeysWithTaskId(tx *dbs.Tx, taskId int64) error {
+	return this.Query(tx).
+		Attr("taskId", taskId).
+		Set("isDone", false).
+		Set("nodes", "{}").
+		Set("errors", "{}").
+		UpdateQuickly()
+}
+
+// CountUserTasksInDay 读取某个用户当前数量
+// day YYYYMMDD
+func (this *HTTPCacheTaskKeyDAO) CountUserTasksInDay(tx *dbs.Tx, userId int64, day string, taskType HTTPCacheTaskType) (int64, error) {
+	if userId <= 0 {
+		return 0, nil
+	}
+
+	// 这里需要包含已删除的
+	return this.Query(tx).
+		Where("taskId IN (SELECT id FROM "+SharedHTTPCacheTaskDAO.Table+" WHERE userId=:userId AND day=:day AND type=:type)").
+		Param("userId", userId).
+		Param("day", day).
+		Param("type", taskType).
+		Count()
+}
+
+// Clean 清理以往的任务
+func (this *HTTPCacheTaskKeyDAO) Clean(tx *dbs.Tx, days int) error {
+	if days <= 0 {
+		days = 30
+	}
+
+	var day = timeutil.Format("Ymd", time.Now().AddDate(0, 0, -days))
+	_, err := this.Query(tx).
+		Where("taskId IN (SELECT id FROM "+SharedHTTPCacheTaskDAO.Table+" WHERE day<=:day)").
+		Param("day", day).
+		Delete()
+	return err
+}

internal/db/models/http_cache_task_key_dao_test.go

@@ -0,0 +1,54 @@
+package models_test
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"testing"
+)
+
+func TestHTTPCacheTaskKeyDAO_CreateKey(t *testing.T) {
+	var dao = models.NewHTTPCacheTaskKeyDAO()
+	var tx *dbs.Tx
+	_, err := dao.CreateKey(tx, 1, "a", "purge", "key", 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestHTTPCacheTaskKeyDAO_UpdateKeyStatus(t *testing.T) {
+	dbs.NotifyReady()
+
+	var dao = models.NewHTTPCacheTaskKeyDAO()
+	var tx *dbs.Tx
+	var errString = "" // "this is error"
+	err := dao.UpdateKeyStatus(tx, 3, 1, errString, []byte(`{"1":true}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestHTTPCacheTaskKeyDAO_CountUserTasksInDay(t *testing.T) {
+	dbs.NotifyReady()
+
+	var dao = models.NewHTTPCacheTaskKeyDAO()
+	var tx *dbs.Tx
+	{
+		count, err := dao.CountUserTasksInDay(tx, 1, timeutil.Format("Ymd"), models.HTTPCacheTaskTypePurge)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("count:", count)
+	}
+	{
+		count, err := dao.CountUserTasksInDay(tx, 1, timeutil.Format("Ymd"), models.HTTPCacheTaskTypeFetch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Log("count:", count)
+	}
+}

internal/db/models/http_cache_task_key_model.go

@@ -0,0 +1,32 @@
+package models
+
+import "github.com/iwind/TeaGo/dbs"
+
+// HTTPCacheTaskKey 缓存任务Key
+type HTTPCacheTaskKey struct {
+	Id        uint64   `field:"id"`        // ID
+	TaskId    uint64   `field:"taskId"`    // 任务ID
+	Key       string   `field:"key"`       // Key
+	KeyType   string   `field:"keyType"`   // Key类型：key|prefix
+	Type      string   `field:"type"`      // 操作类型
+	ClusterId uint32   `field:"clusterId"` // 集群ID
+	Nodes     dbs.JSON `field:"nodes"`     // 节点
+	Errors    dbs.JSON `field:"errors"`    // 错误信息
+	IsDone    bool     `field:"isDone"`    // 是否已完成
+}
+
+type HTTPCacheTaskKeyOperator struct {
+	Id        interface{} // ID
+	TaskId    interface{} // 任务ID
+	Key       interface{} // Key
+	KeyType   interface{} // Key类型：key|prefix
+	Type      interface{} // 操作类型
+	ClusterId interface{} // 集群ID
+	Nodes     interface{} // 节点
+	Errors    interface{} // 错误信息
+	IsDone    interface{} // 是否已完成
+}
+
+func NewHTTPCacheTaskKeyOperator() *HTTPCacheTaskKeyOperator {
+	return &HTTPCacheTaskKeyOperator{}
+}

internal/db/models/http_cache_task_key_model_ext.go

@@ -0,0 +1,20 @@
+package models
+
+import "encoding/json"
+
+// DecodeNodes 解析已完成节点信息
+func (this *HTTPCacheTaskKey) DecodeNodes() map[string]bool {
+	var result = map[string]bool{}
+	var nodesJSON = this.Nodes
+	if IsNull(nodesJSON) {
+		return result
+	}
+
+	err := json.Unmarshal(nodesJSON, &result)
+	if err != nil {
+		// ignore error
+		return result
+	}
+
+	return result
+}

internal/db/models/http_cache_task_model.go

@@ -0,0 +1,36 @@
+package models
+
+// HTTPCacheTask 缓存相关任务
+type HTTPCacheTask struct {
+	Id          uint64 `field:"id"`          // ID
+	UserId      uint32 `field:"userId"`      // 用户ID
+	Type        string `field:"type"`        // 任务类型：purge|fetch
+	KeyType     string `field:"keyType"`     // Key类型
+	State       uint8  `field:"state"`       // 状态
+	CreatedAt   uint64 `field:"createdAt"`   // 创建时间
+	DoneAt      uint64 `field:"doneAt"`      // 完成时间
+	Day         string `field:"day"`         // 创建日期YYYYMMDD
+	IsDone      bool   `field:"isDone"`      // 是否已完成
+	IsOk        bool   `field:"isOk"`        // 是否完全成功
+	IsReady     uint8  `field:"isReady"`     // 是否已准备好
+	Description string `field:"description"` // 描述
+}
+
+type HTTPCacheTaskOperator struct {
+	Id          interface{} // ID
+	UserId      interface{} // 用户ID
+	Type        interface{} // 任务类型：purge|fetch
+	KeyType     interface{} // Key类型
+	State       interface{} // 状态
+	CreatedAt   interface{} // 创建时间
+	DoneAt      interface{} // 完成时间
+	Day         interface{} // 创建日期YYYYMMDD
+	IsDone      interface{} // 是否已完成
+	IsOk        interface{} // 是否完全成功
+	IsReady     interface{} // 是否已准备好
+	Description interface{} // 描述
+}
+
+func NewHTTPCacheTaskOperator() *HTTPCacheTaskOperator {
+	return &HTTPCacheTaskOperator{}
+}

internal/db/models/http_cache_task_model_ext.go

@@ -0,0 +1 @@
+package models

internal/db/models/http_firewall_policy_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
@@ -116,7 +117,7 @@ func (this *HTTPFirewallPolicyDAO) FindAllEnabledFirewallPolicies(tx *dbs.Tx) (r
 
 // CreateFirewallPolicy 创建策略
 func (this *HTTPFirewallPolicyDAO) CreateFirewallPolicy(tx *dbs.Tx, userId int64, serverGroupId int64, serverId int64, isOn bool, name string, description string, inboundJSON []byte, outboundJSON []byte) (int64, error) {
-	op := NewHTTPFirewallPolicyOperator()
+	var op = NewHTTPFirewallPolicyOperator()
 	op.UserId = userId
 	op.GroupId = serverGroupId
 	op.ServerId = serverId
@@ -130,14 +131,31 @@ func (this *HTTPFirewallPolicyDAO) CreateFirewallPolicy(tx *dbs.Tx, userId int64
 	if len(outboundJSON) > 0 {
 		op.Outbound = outboundJSON
 	}
-	op.UseLocalFirewall = true
 
-	{
+	if userId <= 0 && serverGroupId <=0 && serverId <= 0 {
-		synFloodJSON, err := json.Marshal(firewallconfigs.DefaultSYNFloodConfig())
+		// synFlood
+		var synFloodConfig = firewallconfigs.DefaultSYNFloodConfig()
+		synFloodJSON, err := json.Marshal(synFloodConfig)
 		if err != nil {
 			return 0, err
 		}
 		op.SynFlood = synFloodJSON
+
+		// block options
+		var blockOptions = firewallconfigs.DefaultHTTPFirewallBlockAction()
+		blockOptionsJSON, err := json.Marshal(blockOptions)
+		if err != nil {
+			return 0, err
+		}
+		op.BlockOptions = blockOptionsJSON
+
+		// captcha options
+		var captchaOptions = firewallconfigs.DefaultHTTPFirewallCaptchaAction()
+		captchaOptionsJSON, err := json.Marshal(captchaOptions)
+		if err != nil {
+			return 0, err
+		}
+		op.CaptchaOptions = captchaOptionsJSON
 	}
 
 	err := this.Save(tx, op)
@@ -159,8 +177,8 @@ func (this *HTTPFirewallPolicyDAO) CreateDefaultFirewallPolicy(tx *dbs.Tx, name
 		groupCodes = append(groupCodes, group.Code)
 	}
 
-	inboundConfig := &firewallconfigs.HTTPFirewallInboundConfig{IsOn: true}
+	var inboundConfig = &firewallconfigs.HTTPFirewallInboundConfig{IsOn: true}
-	outboundConfig := &firewallconfigs.HTTPFirewallOutboundConfig{IsOn: true}
+	var outboundConfig = &firewallconfigs.HTTPFirewallOutboundConfig{IsOn: true}
 	if templatePolicy.Inbound != nil {
 		for _, group := range templatePolicy.Inbound.Groups {
 			isOn := lists.ContainsString(groupCodes, group.Code)
@@ -206,6 +224,7 @@ func (this *HTTPFirewallPolicyDAO) CreateDefaultFirewallPolicy(tx *dbs.Tx, name
 	if err != nil {
 		return 0, err
 	}
+
 	return policyId, nil
 }
 
@@ -259,11 +278,23 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicyInbound(tx *dbs.Tx, polic
 }
 
 // UpdateFirewallPolicy 修改策略
-func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx, policyId int64, isOn bool, name string, description string, inboundJSON []byte, outboundJSON []byte, blockOptionsJSON []byte, mode firewallconfigs.FirewallMode, useLocalFirewall bool, synFloodConfig *firewallconfigs.SYNFloodConfig) error {
+func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx,
+	policyId int64,
+	isOn bool,
+	name string,
+	description string,
+	inboundJSON []byte,
+	outboundJSON []byte,
+	blockOptionsJSON []byte,
+	captchaOptionsJSON []byte,
+	mode firewallconfigs.FirewallMode,
+	useLocalFirewall bool,
+	synFloodConfig *firewallconfigs.SYNFloodConfig,
+	logConfig *firewallconfigs.HTTPFirewallPolicyLogConfig) error {
 	if policyId <= 0 {
 		return errors.New("invalid policyId")
 	}
-	op := NewHTTPFirewallPolicyOperator()
+	var op = NewHTTPFirewallPolicyOperator()
 	op.Id = policyId
 	op.IsOn = isOn
 	op.Name = name
@@ -279,9 +310,12 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx, policyId int
 	} else {
 		op.Outbound = "null"
 	}
-	if len(blockOptionsJSON) > 0 {
+	if IsNotNull(blockOptionsJSON) {
 		op.BlockOptions = blockOptionsJSON
 	}
+	if IsNotNull(captchaOptionsJSON) {
+		op.CaptchaOptions = captchaOptionsJSON
+	}
 
 	if synFloodConfig != nil {
 		synFloodConfigJSON, err := json.Marshal(synFloodConfig)
@@ -293,6 +327,16 @@ func (this *HTTPFirewallPolicyDAO) UpdateFirewallPolicy(tx *dbs.Tx, policyId int
 		op.SynFlood = "null"
 	}
 
+	if logConfig != nil {
+		logJSON, err := json.Marshal(logConfig)
+		if err != nil {
+			return err
+		}
+		op.Log = logJSON
+	} else {
+		op.Log = "null"
+	}
+
 	op.UseLocalFirewall = useLocalFirewall
 	err := this.Save(tx, op)
 	if err != nil {
@@ -311,7 +355,7 @@ func (this *HTTPFirewallPolicyDAO) CountAllEnabledFirewallPolicies(tx *dbs.Tx, c
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.
 		State(HTTPFirewallPolicyStateEnabled).
@@ -330,7 +374,7 @@ func (this *HTTPFirewallPolicyDAO) ListEnabledFirewallPolicies(tx *dbs.Tx, clust
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(HTTPFirewallPolicyStateEnabled).
@@ -364,7 +408,7 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 		return nil, nil
 	}
 
-	config := &firewallconfigs.HTTPFirewallPolicy{}
+	var config = &firewallconfigs.HTTPFirewallPolicy{}
 	config.Id = int64(policy.Id)
 	config.IsOn = policy.IsOn
 	config.Name = policy.Name
@@ -434,7 +478,7 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 
 	// Block动作配置
 	if IsNotNull(policy.BlockOptions) {
-		blockAction := &firewallconfigs.HTTPFirewallBlockAction{}
+		var blockAction = &firewallconfigs.HTTPFirewallBlockAction{}
 		err = json.Unmarshal(policy.BlockOptions, blockAction)
 		if err != nil {
 			return config, err
@@ -442,6 +486,16 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 		config.BlockOptions = blockAction
 	}
 
+	// Captcha动作配置
+	if IsNotNull(policy.CaptchaOptions) {
+		var captchaAction = &firewallconfigs.HTTPFirewallCaptchaAction{}
+		err = json.Unmarshal(policy.CaptchaOptions, captchaAction)
+		if err != nil {
+			return config, err
+		}
+		config.CaptchaOptions = captchaAction
+	}
+
 	// syn flood
 	if IsNotNull(policy.SynFlood) {
 		var synFloodConfig = &firewallconfigs.SYNFloodConfig{}
@@ -452,6 +506,18 @@ func (this *HTTPFirewallPolicyDAO) ComposeFirewallPolicy(tx *dbs.Tx, policyId in
 		config.SYNFlood = synFloodConfig
 	}
 
+	// log
+	if IsNotNull(policy.Log) {
+		var logConfig = &firewallconfigs.HTTPFirewallPolicyLogConfig{}
+		err = json.Unmarshal(policy.Log, logConfig)
+		if err != nil {
+			return nil, err
+		}
+		config.Log = logConfig
+	} else {
+		config.Log = firewallconfigs.DefaultHTTPFirewallPolicyLogConfig
+	}
+
 	if cacheMap != nil {
 		cacheMap.Put(cacheKey, config)
 	}
@@ -492,6 +558,7 @@ func (this *HTTPFirewallPolicyDAO) CheckUserFirewallPolicy(tx *dbs.Tx, userId in
 }
 
 // FindEnabledFirewallPolicyIdsWithIPListId 查找包含某个IPList的所有策略
+// TODO 改成通过 serverId 查询
 func (this *HTTPFirewallPolicyDAO) FindEnabledFirewallPolicyIdsWithIPListId(tx *dbs.Tx, ipListId int64) ([]int64, error) {
 	ones, err := this.Query(tx).
 		ResultPk().
@@ -510,6 +577,7 @@ func (this *HTTPFirewallPolicyDAO) FindEnabledFirewallPolicyIdsWithIPListId(tx *
 }
 
 // FindEnabledFirewallPolicyWithIPListId 查找使用某个IPList的策略
+// TODO 改成通过 serverId 查询
 func (this *HTTPFirewallPolicyDAO) FindEnabledFirewallPolicyWithIPListId(tx *dbs.Tx, ipListId int64) (*HTTPFirewallPolicy, error) {
 	one, err := this.Query(tx).
 		State(HTTPFirewallPolicyStateEnabled).

internal/db/models/http_firewall_policy_model.go

@@ -18,9 +18,11 @@ type HTTPFirewallPolicy struct {
 	Inbound          dbs.JSON `field:"inbound"`          // 入站规则
 	Outbound         dbs.JSON `field:"outbound"`         // 出站规则
 	BlockOptions     dbs.JSON `field:"blockOptions"`     // BLOCK选项
+	CaptchaOptions   dbs.JSON `field:"captchaOptions"`   // 验证码选项
 	Mode             string   `field:"mode"`             // 模式
 	UseLocalFirewall uint8    `field:"useLocalFirewall"` // 是否自动使用本地防火墙
 	SynFlood         dbs.JSON `field:"synFlood"`         // SynFlood防御设置
+	Log              dbs.JSON `field:"log"`              // 日志配置
 }
 
 type HTTPFirewallPolicyOperator struct {
@@ -38,9 +40,11 @@ type HTTPFirewallPolicyOperator struct {
 	Inbound          interface{} // 入站规则
 	Outbound         interface{} // 出站规则
 	BlockOptions     interface{} // BLOCK选项
+	CaptchaOptions   interface{} // 验证码选项
 	Mode             interface{} // 模式
 	UseLocalFirewall interface{} // 是否自动使用本地防火墙
 	SynFlood         interface{} // SynFlood防御设置
+	Log              interface{} // 日志配置
 }
 
 func NewHTTPFirewallPolicyOperator() *HTTPFirewallPolicyOperator {

internal/db/models/http_page_dao.go

@@ -77,8 +77,9 @@ func (this *HTTPPageDAO) FindEnabledHTTPPage(tx *dbs.Tx, id int64) (*HTTPPage, e
 }
 
 // CreatePage 创建Page
-func (this *HTTPPageDAO) CreatePage(tx *dbs.Tx, statusList []string, bodyType shared.BodyType, url string, body string, newStatus int) (pageId int64, err error) {
+func (this *HTTPPageDAO) CreatePage(tx *dbs.Tx, userId int64, statusList []string, bodyType shared.BodyType, url string, body string, newStatus int) (pageId int64, err error) {
 	op := NewHTTPPageOperator()
+	op.UserId = userId
 	op.IsOn = true
 	op.State = HTTPPageStateEnabled
 
@@ -182,6 +183,26 @@ func (this *HTTPPageDAO) ComposePageConfig(tx *dbs.Tx, pageId int64, cacheMap *u
 	return config, nil
 }
 
+// CheckUserPage 检查用户页面
+func (this *HTTPPageDAO) CheckUserPage(tx *dbs.Tx, userId int64, pageId int64) error {
+	if userId <= 0 || pageId <= 0 {
+		return ErrNotFound
+	}
+
+	b, err := this.Query(tx).
+		Pk(pageId).
+		Attr("userId", userId).
+		State(HTTPPageStateEnabled).
+		Exist()
+	if err != nil {
+		return err
+	}
+	if !b {
+		return ErrNotFound
+	}
+	return nil
+}
+
 // NotifyUpdate 通知更新
 func (this *HTTPPageDAO) NotifyUpdate(tx *dbs.Tx, pageId int64) error {
 	webId, err := SharedHTTPWebDAO.FindEnabledWebIdWithPageId(tx, pageId)

internal/db/models/http_web_dao.go

@@ -3,6 +3,7 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
@@ -94,7 +95,7 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64, cacheMap *util
 		return nil, nil
 	}
 
-	config := &serverconfigs.HTTPWebConfig{}
+	var config = &serverconfigs.HTTPWebConfig{}
 	config.Id = webId
 	config.IsOn = web.IsOn
 
@@ -446,6 +447,16 @@ func (this *HTTPWebDAO) ComposeWebConfig(tx *dbs.Tx, webId int64, cacheMap *util
 		}
 	}
 
+	// UAM
+	if teaconst.IsPlus && IsNotNull(web.Uam) {
+		var uamConfig = &serverconfigs.UAMConfig{}
+		err = json.Unmarshal(web.Uam, uamConfig)
+		if err != nil {
+			return nil, err
+		}
+		config.UAM = uamConfig
+	}
+
 	if cacheMap != nil {
 		cacheMap.Put(cacheKey, config)
 	}
@@ -1168,6 +1179,35 @@ func (this *HTTPWebDAO) FindWebRequestScripts(tx *dbs.Tx, webId int64) (*serverc
 	return config, nil
 }
 
+// UpdateWebUAM 开启UAM
+func (this *HTTPWebDAO) UpdateWebUAM(tx *dbs.Tx, webId int64, uamConfig *serverconfigs.UAMConfig) error {
+	if uamConfig == nil {
+		return nil
+	}
+	configJSON, err := json.Marshal(uamConfig)
+	if err != nil {
+		return err
+	}
+
+	err = this.Query(tx).
+		Pk(webId).
+		Set("uam", configJSON).
+		UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	return this.NotifyUpdate(tx, webId)
+}
+
+// FindWebUAM 查找服务的UAM配置
+func (this *HTTPWebDAO) FindWebUAM(tx *dbs.Tx, webId int64) ([]byte, error) {
+	return this.Query(tx).
+		Pk(webId).
+		Result("uam").
+		FindJSONCol()
+}
+
 // NotifyUpdate 通知更新
 func (this *HTTPWebDAO) NotifyUpdate(tx *dbs.Tx, webId int64) error {
 	// server

internal/db/models/http_web_model.go

@@ -37,6 +37,7 @@ type HTTPWeb struct {
 	MergeSlashes       uint8    `field:"mergeSlashes"`       // 是否合并路径中的斜杠
 	RequestLimit       dbs.JSON `field:"requestLimit"`       // 请求限制
 	RequestScripts     dbs.JSON `field:"requestScripts"`     // 请求脚本
+	Uam                dbs.JSON `field:"uam"`                // UAM设置
 }
 
 type HTTPWebOperator struct {
@@ -73,6 +74,7 @@ type HTTPWebOperator struct {
 	MergeSlashes       interface{} // 是否合并路径中的斜杠
 	RequestLimit       interface{} // 请求限制
 	RequestScripts     interface{} // 请求脚本
+	Uam                interface{} // UAM设置
 }
 
 func NewHTTPWebOperator() *HTTPWebOperator {

internal/db/models/ip_item_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
@@ -92,6 +93,71 @@ func (this *IPItemDAO) DisableIPItem(tx *dbs.Tx, id int64) error {
 	return this.NotifyUpdate(tx, id)
 }
 
+// DisableIPItemsWithIP 禁用某个IP相关条目
+func (this *IPItemDAO) DisableIPItemsWithIP(tx *dbs.Tx, ipFrom string, ipTo string, userId int64, listId int64) error {
+	if len(ipFrom) == 0 {
+		return errors.New("invalid 'ipFrom'")
+	}
+
+	var query = this.Query(tx).
+		Result("id", "listId").
+		Attr("ipFrom", ipFrom).
+		Attr("ipTo", ipTo).
+		State(IPItemStateEnabled)
+
+	if listId > 0 {
+		if userId > 0 {
+			err := SharedIPListDAO.CheckUserIPList(tx, userId, listId)
+			if err != nil {
+				return err
+			}
+		}
+
+		query.Attr("listId", listId)
+	}
+
+	ones, err := query.FindAll()
+	if err != nil {
+		return err
+	}
+
+	var itemIds = []int64{}
+	for _, one := range ones {
+		var item = one.(*IPItem)
+		var itemId = int64(item.Id)
+		var itemListId = int64(item.ListId)
+		if itemListId != listId && userId > 0 {
+			err = SharedIPListDAO.CheckUserIPList(tx, userId, itemListId)
+			if err != nil {
+				// ignore error
+				continue
+			}
+		}
+		itemIds = append(itemIds, itemId)
+	}
+
+	for _, itemId := range itemIds {
+		version, err := SharedIPListDAO.IncreaseVersion(tx)
+		if err != nil {
+			return err
+		}
+
+		_, err = this.Query(tx).
+			Pk(itemId).
+			Set("state", IPItemStateDisabled).
+			Set("version", version).
+			Update()
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(itemIds) > 0 {
+		return this.NotifyUpdate(tx, itemIds[len(itemIds)-1])
+	}
+	return nil
+}
+
 // DisableIPItemsWithListId 禁用某个IP名单内的所有IP
 func (this *IPItemDAO) DisableIPItemsWithListId(tx *dbs.Tx, listId int64) error {
 	for {
@@ -141,14 +207,35 @@ func (this *IPItemDAO) FindEnabledIPItem(tx *dbs.Tx, id int64) (*IPItem, error)
 
 // DeleteOldItem 根据IP删除以前的旧记录
 func (this *IPItemDAO) DeleteOldItem(tx *dbs.Tx, listId int64, ipFrom string, ipTo string) error {
-	_, err := this.Query(tx).
+	ones, err := this.Query(tx).
+		ResultPk().
 		UseIndex("ipFrom").
 		Attr("listId", listId).
 		Attr("ipFrom", ipFrom).
 		Attr("ipTo", ipTo).
-		Delete()
+		Set("state", IPItemStateEnabled).
-	// 这里不通知更新
+		FindAll()
-	return err
+	if err != nil {
+		return err
+	}
+	for _, one := range ones {
+		var itemId = int64(one.(*IPItem).Id)
+		version, err := SharedIPListDAO.IncreaseVersion(tx)
+		if err != nil {
+			return err
+		}
+
+		err = this.Query(tx).
+			Pk(itemId).
+			Set("version", version).
+			Set("state", IPItemStateDisabled).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 // CreateIPItem 创建IP
@@ -201,6 +288,8 @@ func (this *IPItemDAO) CreateIPItem(tx *dbs.Tx,
 	}
 
 	op.State = IPItemStateEnabled
+	op.UpdatedAt = time.Now().Unix()
+
 	err = this.Save(tx, op)
 	if err != nil {
 		return 0, err
@@ -264,13 +353,13 @@ func (this *IPItemDAO) UpdateIPItem(tx *dbs.Tx, itemId int64, ipFrom string, ipT
 }
 
 // CountIPItemsWithListId 计算IP数量
-func (this *IPItemDAO) CountIPItemsWithListId(tx *dbs.Tx, listId int64, ipFrom string, ipTo string, keyword string) (int64, error) {
+func (this *IPItemDAO) CountIPItemsWithListId(tx *dbs.Tx, listId int64, ipFrom string, ipTo string, keyword string, eventLevel string) (int64, error) {
 	var query = this.Query(tx).
 		State(IPItemStateEnabled).
 		Attr("listId", listId)
 	if len(keyword) > 0 {
 		query.Where("(ipFrom LIKE :keyword OR ipTo LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(ipFrom) > 0 {
 		query.Attr("ipFrom", ipFrom)
@@ -278,17 +367,20 @@ func (this *IPItemDAO) CountIPItemsWithListId(tx *dbs.Tx, listId int64, ipFrom s
 	if len(ipTo) > 0 {
 		query.Attr("ipTo", ipTo)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
 	return query.Count()
 }
 
 // ListIPItemsWithListId 查找IP列表
-func (this *IPItemDAO) ListIPItemsWithListId(tx *dbs.Tx, listId int64, keyword string, ipFrom string, ipTo string, offset int64, size int64) (result []*IPItem, err error) {
+func (this *IPItemDAO) ListIPItemsWithListId(tx *dbs.Tx, listId int64, keyword string, ipFrom string, ipTo string, eventLevel string, offset int64, size int64) (result []*IPItem, err error) {
 	var query = this.Query(tx).
 		State(IPItemStateEnabled).
 		Attr("listId", listId)
 	if len(keyword) > 0 {
 		query.Where("(ipFrom LIKE :keyword OR ipTo LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(ipFrom) > 0 {
 		query.Attr("ipFrom", ipFrom)
@@ -296,6 +388,9 @@ func (this *IPItemDAO) ListIPItemsWithListId(tx *dbs.Tx, listId int64, keyword s
 	if len(ipTo) > 0 {
 		query.Attr("ipTo", ipTo)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
 	_, err = query.
 		DescPk().
 		Slice(&result).
@@ -371,7 +466,7 @@ func (this *IPItemDAO) ExistsEnabledItem(tx *dbs.Tx, itemId int64) (bool, error)
 }
 
 // CountAllEnabledIPItems 计算数量
-func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool) (int64, error) {
+func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool, eventLevel string, listType string) (int64, error) {
 	var query = this.Query(tx)
 	if len(ip) > 0 {
 		query.Attr("ipFrom", ip)
@@ -379,11 +474,20 @@ func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int6
 	if listId > 0 {
 		query.Attr("listId", listId)
 	} else {
-		query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		if len(listType) > 0 {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1 AND type=:listType))")
+			query.Param("listType", listType)
+		} else {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		}
 	}
 	if unread {
 		query.Attr("isRead", 0)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
+
 	return query.
 		State(IPItemStateEnabled).
 		Where("(expiredAt=0 OR expiredAt>:expiredAt)").
@@ -392,7 +496,7 @@ func (this *IPItemDAO) CountAllEnabledIPItems(tx *dbs.Tx, ip string, listId int6
 }
 
 // ListAllEnabledIPItems 搜索所有IP
-func (this *IPItemDAO) ListAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool, offset int64, size int64) (result []*IPItem, err error) {
+func (this *IPItemDAO) ListAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64, unread bool, eventLevel string, listType string, offset int64, size int64) (result []*IPItem, err error) {
 	var query = this.Query(tx)
 	if len(ip) > 0 {
 		query.Attr("ipFrom", ip)
@@ -400,11 +504,19 @@ func (this *IPItemDAO) ListAllEnabledIPItems(tx *dbs.Tx, ip string, listId int64
 	if listId > 0 {
 		query.Attr("listId", listId)
 	} else {
-		query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		if len(listType) > 0 {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1 AND type=:listType))")
+			query.Param("listType", listType)
+		} else {
+			query.Where("(listId=" + types.String(firewallconfigs.GlobalListId) + " OR listId IN (SELECT id FROM " + SharedIPListDAO.Table + " WHERE state=1))")
+		}
 	}
 	if unread {
 		query.Attr("isRead", 0)
 	}
+	if len(eventLevel) > 0 {
+		query.Attr("eventLevel", eventLevel)
+	}
 	_, err = query.
 		State(IPItemStateEnabled).
 		Where("(expiredAt=0 OR expiredAt>:expiredAt)").
@@ -429,7 +541,7 @@ func (this *IPItemDAO) UpdateItemsRead(tx *dbs.Tx) error {
 func (this *IPItemDAO) CleanExpiredIPItems(tx *dbs.Tx) error {
 	// 删除 N 天之前过期的数据
 	_, err := this.Query(tx).
-		Where("expiredAt<=:timestamp").
+		Where("(createdAt<=:timestamp AND updatedAt<=:timestamp)").
 		State(IPItemStateDisabled).
 		Param("timestamp", time.Now().Unix()-7*86400). // N 天之前过期的
 		Limit(10000).                                  // 限制条数，防止数量过多导致超时

internal/db/models/ip_item_dao_test.go

@@ -1,6 +1,7 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/firewallconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/dbs"
@@ -14,7 +15,7 @@ func TestIPItemDAO_NotifyClustersUpdate(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	err := SharedIPItemDAO.NotifyUpdate(tx, 28)
+	err := models.SharedIPItemDAO.NotifyUpdate(tx, 28)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -25,7 +26,7 @@ func TestIPItemDAO_DisableIPItemsWithListId(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	err := SharedIPItemDAO.DisableIPItemsWithListId(tx, 67)
+	err := models.SharedIPItemDAO.DisableIPItemsWithListId(tx, 67)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -36,7 +37,7 @@ func TestIPItemDAO_ListIPItemsAfterVersion(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	_, err := SharedIPItemDAO.ListIPItemsAfterVersion(tx, 0, 100)
+	_, err := models.SharedIPItemDAO.ListIPItemsAfterVersion(tx, 0, 100)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -47,10 +48,10 @@ func TestIPItemDAO_CreateManyIPs(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	var dao = NewIPItemDAO()
+	var dao = models.NewIPItemDAO()
 	var n = 10
 	for i := 0; i < n; i++ {
-		itemId, err := dao.CreateIPItem(tx, firewallconfigs.GlobalListId, "192."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255)), "", time.Now().Unix()+86400, "test", IPItemTypeIPv4, "warning", 0, 0, 0, 0, 0, 0, 0)
+		itemId, err := dao.CreateIPItem(tx, firewallconfigs.GlobalListId, "192."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255))+"."+types.String(rands.Int(0, 255)), "", time.Now().Unix()+86400, "test", models.IPItemTypeIPv4, "warning", 0, 0, 0, 0, 0, 0, 0)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -62,3 +63,14 @@ func TestIPItemDAO_CreateManyIPs(t *testing.T) {
 	}
 	t.Log("ok")
 }
+
+func TestIPItemDAO_DisableIPItemsWithIP(t *testing.T) {
+	dbs.NotifyReady()
+
+	var tx *dbs.Tx
+	err := models.SharedIPItemDAO.DisableIPItemsWithIP(tx, "192.168.1.100", "", 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}

internal/db/models/ip_list_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
@@ -137,10 +138,11 @@ func (this *IPListDAO) FindIPListCacheable(tx *dbs.Tx, listId int64) (*IPList, e
 }
 
 // CreateIPList 创建名单
-func (this *IPListDAO) CreateIPList(tx *dbs.Tx, userId int64, listType ipconfigs.IPListType, name string, code string, timeoutJSON []byte, description string, isPublic bool, isGlobal bool) (int64, error) {
+func (this *IPListDAO) CreateIPList(tx *dbs.Tx, userId int64, serverId int64, listType ipconfigs.IPListType, name string, code string, timeoutJSON []byte, description string, isPublic bool, isGlobal bool) (int64, error) {
-	op := NewIPListOperator()
+	var op = NewIPListOperator()
 	op.IsOn = true
 	op.UserId = userId
+	op.ServerId = serverId
 	op.State = IPListStateEnabled
 	op.Type = listType
 	op.Name = name
@@ -188,26 +190,25 @@ func (this *IPListDAO) CheckUserIPList(tx *dbs.Tx, userId int64, listId int64) e
 		return ErrNotFound
 	}
 
-	ok, err := this.Query(tx).
+	// 获取名单信息
+	listOne, err := this.Query(tx).
 		Pk(listId).
-		Attr("userId", userId).
+		Result("userId", "serverId").
-		Exist()
+		Find()
 	if err != nil {
 		return err
 	}
-	if ok {
+	if listOne == nil {
+		return ErrNotFound
+	}
+	var list = listOne.(*IPList)
+	if int64(list.UserId) == userId {
 		return nil
 	}
 
-	// 检查是否被用户的服务所使用
+	var serverId = int64(list.ServerId)
-	policyIds, err := SharedHTTPFirewallPolicyDAO.FindEnabledFirewallPolicyIdsWithIPListId(tx, listId)
+	if serverId > 0 {
-	if err != nil {
+		return SharedServerDAO.CheckUserServer(tx, userId, serverId)
-		return err
-	}
-	for _, policyId := range policyIds {
-		if SharedHTTPFirewallPolicyDAO.CheckUserFirewallPolicy(tx, userId, policyId) == nil {
-			return nil
-		}
 	}
 
 	return ErrNotFound
@@ -221,7 +222,7 @@ func (this *IPListDAO) CountAllEnabledIPLists(tx *dbs.Tx, listType string, isPub
 		Attr("isPublic", isPublic)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.Count()
 }
@@ -234,7 +235,7 @@ func (this *IPListDAO) ListEnabledIPLists(tx *dbs.Tx, listType string, isPublic
 		Attr("isPublic", isPublic)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.Offset(offset).
 		Limit(size).

internal/db/models/ip_list_dao_test.go

@@ -20,6 +20,39 @@ func TestIPListDAO_IncreaseVersion(t *testing.T) {
 	t.Log("version:", version)
 }
 
+func TestIPListDAO_CheckUserIPList(t *testing.T) {
+	dbs.NotifyReady()
+
+	var tx *dbs.Tx
+
+	{
+		err := NewIPListDAO().CheckUserIPList(tx, 1, 100)
+		if err == ErrNotFound {
+			t.Log("not found")
+		} else {
+			t.Log(err)
+		}
+	}
+
+	{
+		err := NewIPListDAO().CheckUserIPList(tx, 1, 85)
+		if err == ErrNotFound {
+			t.Log("not found")
+		} else {
+			t.Log(err)
+		}
+	}
+
+	{
+		err := NewIPListDAO().CheckUserIPList(tx, 1, 17)
+		if err == ErrNotFound {
+			t.Log("not found")
+		} else {
+			t.Log(err)
+		}
+	}
+}
+
 func BenchmarkIPListDAO_IncreaseVersion(b *testing.B) {
 	runtime.GOMAXPROCS(1)
 
@@ -32,3 +65,4 @@ func BenchmarkIPListDAO_IncreaseVersion(b *testing.B) {
 		_, _ = dao.IncreaseVersion(tx)
 	}
 }
+

internal/db/models/ip_list_model.go

@@ -9,6 +9,7 @@ type IPList struct {
 	Type        string   `field:"type"`        // 类型
 	AdminId     uint32   `field:"adminId"`     // 用户ID
 	UserId      uint32   `field:"userId"`      // 用户ID
+	ServerId    uint64   `field:"serverId"`    // 服务ID
 	Name        string   `field:"name"`        // 列表名
 	Code        string   `field:"code"`        // 代号
 	State       uint8    `field:"state"`       // 状态
@@ -26,6 +27,7 @@ type IPListOperator struct {
 	Type        interface{} // 类型
 	AdminId     interface{} // 用户ID
 	UserId      interface{} // 用户ID
+	ServerId    interface{} // 服务ID
 	Name        interface{} // 列表名
 	Code        interface{} // 代号
 	State       interface{} // 状态

internal/db/models/log_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
@@ -72,7 +73,7 @@ func (this *LogDAO) CountLogs(tx *dbs.Tx, dayFrom string, dayTo string, keyword
 	}
 	if len(keyword) > 0 {
 		query.Where("(description LIKE :keyword OR ip LIKE :keyword OR action LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	// 用户类型
@@ -100,7 +101,7 @@ func (this *LogDAO) ListLogs(tx *dbs.Tx, offset int64, size int64, dayFrom strin
 	}
 	if len(keyword) > 0 {
 		query.Where("(description LIKE :keyword OR ip LIKE :keyword OR action LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	// 用户类型

internal/db/models/message_media_instance_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	_ "github.com/go-sql-driver/mysql"
@@ -149,7 +150,7 @@ func (this *MessageMediaInstanceDAO) CountAllEnabledMediaInstances(tx *dbs.Tx, m
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.
 		State(MessageMediaInstanceStateEnabled).
@@ -165,7 +166,7 @@ func (this *MessageMediaInstanceDAO) ListAllEnabledMediaInstances(tx *dbs.Tx, me
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(MessageMediaInstanceStateEnabled).

internal/db/models/message_recipient_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
@@ -172,7 +173,7 @@ func (this *MessageRecipientDAO) CountAllEnabledRecipients(tx *dbs.Tx, adminId i
 	}
 	if len(keyword) > 0 {
 		query.Where("(`user` LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.
 		State(MessageRecipientStateEnabled).
@@ -197,7 +198,7 @@ func (this *MessageRecipientDAO) ListAllEnabledRecipients(tx *dbs.Tx, adminId in
 	}
 	if len(keyword) > 0 {
 		query.Where("(`user` LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(MessageRecipientStateEnabled).

internal/db/models/metric_item_dao.go

@@ -12,6 +12,7 @@ import (
 	"github.com/iwind/TeaGo/types"
 	"sort"
 	"strings"
+	"sync"
 )
 
 const (
@@ -34,6 +35,9 @@ func NewMetricItemDAO() *MetricItemDAO {
 
 var SharedMetricItemDAO *MetricItemDAO
 
+var metricItemLastTimeCacheMap = map[int64]string{} // itemId => time
+var metricItemLastTimeCacheLocker = &sync.Mutex{}
+
 func init() {
 	dbs.OnReady(func() {
 		SharedMetricItemDAO = NewMetricItemDAO()
@@ -108,10 +112,10 @@ func (this *MetricItemDAO) FindMetricItemName(tx *dbs.Tx, id int64) (string, err
 }
 
 // CreateItem 创建指标
-func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string, name string, keys []string, period int32, periodUnit string, value string, isPublic bool) (int64, error) {
+func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string, name string, keys []string, period int32, periodUnit string, expiresPeriod int32, value string, isPublic bool) (int64, error) {
 	sort.Strings(keys)
 
-	op := NewMetricItemOperator()
+	var op = NewMetricItemOperator()
 	op.Code = code
 	op.Category = category
 	op.Name = name
@@ -126,6 +130,7 @@ func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string,
 	}
 	op.Period = period
 	op.PeriodUnit = periodUnit
+	op.ExpiresPeriod = expiresPeriod
 	op.Value = value
 	op.IsPublic = isPublic
 	op.IsOn = true
@@ -146,7 +151,7 @@ func (this *MetricItemDAO) CreateItem(tx *dbs.Tx, code string, category string,
 }
 
 // UpdateItem 修改\指标
-func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, keys []string, period int32, periodUnit string, value string, isOn bool, isPublic bool) error {
+func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, keys []string, period int32, periodUnit string, expiresPeriod int32, value string, isOn bool, isPublic bool) error {
 	if itemId <= 0 {
 		return errors.New("invalid itemId")
 	}
@@ -168,7 +173,7 @@ func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, key
 	}
 
 	// 保存
-	op := NewMetricItemOperator()
+	var op = NewMetricItemOperator()
 	op.Id = itemId
 	op.Name = name
 	if len(keys) > 0 {
@@ -182,6 +187,7 @@ func (this *MetricItemDAO) UpdateItem(tx *dbs.Tx, itemId int64, name string, key
 	}
 	op.Period = period
 	op.PeriodUnit = periodUnit
+	op.ExpiresPeriod = expiresPeriod
 	op.Value = value
 	op.IsOn = isOn
 	if versionChanged {
@@ -282,14 +288,15 @@ func (this *MetricItemDAO) ComposeItemConfig(tx *dbs.Tx, itemId int64) (*serverc
 	}
 	var item = one.(*MetricItem)
 	var config = &serverconfigs.MetricItemConfig{
-		Id:         int64(item.Id),
+		Id:            int64(item.Id),
-		IsOn:       item.IsOn,
+		IsOn:          item.IsOn,
-		Period:     types.Int(item.Period),
+		Period:        types.Int(item.Period),
-		PeriodUnit: item.PeriodUnit,
+		PeriodUnit:    item.PeriodUnit,
-		Category:   item.Category,
+		ExpiresPeriod: types.Int(item.ExpiresPeriod),
-		Value:      item.Value,
+		Category:      item.Category,
-		Keys:       item.DecodeKeys(),
+		Value:         item.Value,
-		Version:    types.Int32(item.Version),
+		Keys:          item.DecodeKeys(),
+		Version:       types.Int32(item.Version),
 	}
 
 	return config, nil
@@ -301,14 +308,15 @@ func (this *MetricItemDAO) ComposeItemConfigWithItem(item *MetricItem) *serverco
 		return nil
 	}
 	var config = &serverconfigs.MetricItemConfig{
-		Id:         int64(item.Id),
+		Id:            int64(item.Id),
-		IsOn:       item.IsOn,
+		IsOn:          item.IsOn,
-		Period:     types.Int(item.Period),
+		Period:        types.Int(item.Period),
-		PeriodUnit: item.PeriodUnit,
+		PeriodUnit:    item.PeriodUnit,
-		Category:   item.Category,
+		ExpiresPeriod: types.Int(item.ExpiresPeriod),
-		Value:      item.Value,
+		Category:      item.Category,
-		Keys:       item.DecodeKeys(),
+		Value:         item.Value,
-		Version:    types.Int32(item.Version),
+		Keys:          item.DecodeKeys(),
+		Version:       types.Int32(item.Version),
 	}
 
 	return config
@@ -326,6 +334,32 @@ func (this *MetricItemDAO) FindItemVersion(tx *dbs.Tx, itemId int64) (int32, err
 	return types.Int32(version), nil
 }
 
+// UpdateMetricLastTime 更新指标最新数据的时间
+func (this *MetricItemDAO) UpdateMetricLastTime(tx *dbs.Tx, itemId int64, lastTime string) error {
+	metricItemLastTimeCacheLocker.Lock()
+	cachedTime, ok := metricItemLastTimeCacheMap[itemId]
+	if ok && cachedTime == lastTime {
+		metricItemLastTimeCacheLocker.Unlock()
+		return nil
+	}
+
+	metricItemLastTimeCacheMap[itemId] = lastTime
+	metricItemLastTimeCacheLocker.Unlock()
+
+	return this.Query(tx).
+		Pk(itemId).
+		Set("lastTime", lastTime).
+		UpdateQuickly()
+}
+
+// FindMetricLastTime 读取指标最新数据的时间
+func (this *MetricItemDAO) FindMetricLastTime(tx *dbs.Tx, itemId int64) (string, error) {
+	return this.Query(tx).
+		Result("lastTime").
+		Pk(itemId).
+		FindStringCol("")
+}
+
 // NotifyUpdate 通知更新
 func (this *MetricItemDAO) NotifyUpdate(tx *dbs.Tx, itemId int64, isPublic bool) error {
 	if isPublic {

internal/db/models/metric_item_dao_test.go

@@ -1,6 +1,13 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
+	"testing"
 )
+
+func TestMetricStatDAO_Clean(t *testing.T) {
+	var dao = models.NewMetricStatDAO()
+	t.Log(dao.Clean(nil))
+}

internal/db/models/metric_item_model.go

@@ -4,37 +4,41 @@ import "github.com/iwind/TeaGo/dbs"
 
 // MetricItem 指标定义
 type MetricItem struct {
-	Id         uint64   `field:"id"`         // ID
+	Id            uint64   `field:"id"`            // ID
-	IsOn       bool     `field:"isOn"`       // 是否启用
+	IsOn          bool     `field:"isOn"`          // 是否启用
-	Code       string   `field:"code"`       // 代号（用来区分是否内置）
+	Code          string   `field:"code"`          // 代号（用来区分是否内置）
-	Category   string   `field:"category"`   // 类型，比如http, tcp等
+	Category      string   `field:"category"`      // 类型，比如http, tcp等
-	AdminId    uint32   `field:"adminId"`    // 管理员ID
+	AdminId       uint32   `field:"adminId"`       // 管理员ID
-	UserId     uint32   `field:"userId"`     // 用户ID
+	UserId        uint32   `field:"userId"`        // 用户ID
-	Name       string   `field:"name"`       // 指标名称
+	Name          string   `field:"name"`          // 指标名称
-	Keys       dbs.JSON `field:"keys"`       // 统计的Key
+	Keys          dbs.JSON `field:"keys"`          // 统计的Key
-	Period     uint32   `field:"period"`     // 周期
+	Period        uint32   `field:"period"`        // 周期
-	PeriodUnit string   `field:"periodUnit"` // 周期单位
+	PeriodUnit    string   `field:"periodUnit"`    // 周期单位
-	Value      string   `field:"value"`      // 值运算
+	ExpiresPeriod uint32   `field:"expiresPeriod"` // 过期周期
-	State      uint8    `field:"state"`      // 状态
+	Value         string   `field:"value"`         // 值运算
-	Version    uint32   `field:"version"`    // 版本号
+	State         uint8    `field:"state"`         // 状态
-	IsPublic   bool     `field:"isPublic"`   // 是否为公用
+	Version       uint32   `field:"version"`       // 版本号
+	IsPublic      bool     `field:"isPublic"`      // 是否为公用
+	LastTime      string   `field:"lastTime"`      // 最新时间
 }
 
 type MetricItemOperator struct {
-	Id         interface{} // ID
+	Id            interface{} // ID
-	IsOn       interface{} // 是否启用
+	IsOn          interface{} // 是否启用
-	Code       interface{} // 代号（用来区分是否内置）
+	Code          interface{} // 代号（用来区分是否内置）
-	Category   interface{} // 类型，比如http, tcp等
+	Category      interface{} // 类型，比如http, tcp等
-	AdminId    interface{} // 管理员ID
+	AdminId       interface{} // 管理员ID
-	UserId     interface{} // 用户ID
+	UserId        interface{} // 用户ID
-	Name       interface{} // 指标名称
+	Name          interface{} // 指标名称
-	Keys       interface{} // 统计的Key
+	Keys          interface{} // 统计的Key
-	Period     interface{} // 周期
+	Period        interface{} // 周期
-	PeriodUnit interface{} // 周期单位
+	PeriodUnit    interface{} // 周期单位
-	Value      interface{} // 值运算
+	ExpiresPeriod interface{} // 过期周期
-	State      interface{} // 状态
+	Value         interface{} // 值运算
-	Version    interface{} // 版本号
+	State         interface{} // 状态
-	IsPublic   interface{} // 是否为公用
+	Version       interface{} // 版本号
+	IsPublic      interface{} // 是否为公用
+	LastTime      interface{} // 最新时间
 }
 
 func NewMetricItemOperator() *MetricItemOperator {

internal/db/models/metric_stat_dao.go

@@ -3,16 +3,21 @@ package models
 import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
 	"github.com/iwind/TeaGo/lists"
-	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sort"
 	"strconv"
+	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -26,13 +31,15 @@ func init() {
 			for range ticker.C {
 				err := SharedMetricStatDAO.Clean(nil)
 				if err != nil {
-					logs.Println("SharedMetricStatDAO: clean expired data failed: " + err.Error())
+					remotelogs.Error("SharedMetricStatDAO", "clean expired data failed: "+err.Error())
 				}
 			}
 		})
 	})
 }
 
+const MetricStatTablePartials = 20 // 表格Partial数量
+
 func NewMetricStatDAO() *MetricStatDAO {
 	return dbs.NewDAO(&MetricStatDAO{
 		DAOObject: dbs.DAOObject{
@@ -65,7 +72,8 @@ func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64,
 	} else {
 		keysString = "[]"
 	}
-	return this.Query(tx).
+	err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Param("value", value).
 		InsertOrUpdateQuickly(maps.Map{
 			"hash":       hash,
@@ -81,57 +89,135 @@ func (this *MetricStatDAO) CreateStat(tx *dbs.Tx, hash string, clusterId int64,
 		}, maps.Map{
 			"value": value,
 		})
+	if err != nil {
+		if this.canIgnore(err) {
+			return nil
+		}
+		return err
+	}
+
+	return SharedMetricItemDAO.UpdateMetricLastTime(tx, itemId, time)
 }
 
 // DeleteOldVersionItemStats 删除以前版本的统计数据
 func (this *MetricStatDAO) DeleteOldVersionItemStats(tx *dbs.Tx, itemId int64, version int32) error {
-	_, err := this.Query(tx).
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("itemId", itemId).
+		_, err := this.Query(tx).
-		Where("version<:version").
+			Table(table).
-		Param("version", version).
+			Attr("itemId", itemId).
-		Delete()
+			Where("version<:version").
-	return err
+			Param("version", version).
+			Delete()
+		return err
+	})
 }
 
 // DeleteItemStats 删除某个指标相关的统计数据
 func (this *MetricStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error {
-	_, err := this.Query(tx).
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("itemId", itemId).
+		_, err := this.Query(tx).
-		Delete()
+			Table(table).
-	return err
+			Attr("itemId", itemId).
+			Delete()
+		return err
+	})
 }
 
 // DeleteNodeItemStats 删除某个节点的统计数据
 func (this *MetricStatDAO) DeleteNodeItemStats(tx *dbs.Tx, nodeId int64, serverId int64, itemId int64, time string) error {
-	_, err := this.Query(tx).
+	if serverId > 0 {
-		Attr("nodeId", nodeId).
+		_, err := this.Query(tx).
-		Attr("serverId", serverId).
+			Table(this.partialTable(serverId)).
-		Attr("itemId", itemId).
+			Attr("nodeId", nodeId).
-		Attr("time", time).
+			Attr("serverId", serverId).
-		Delete()
+			Attr("itemId", itemId).
+			Attr("time", time).
+			Delete()
+		if this.canIgnore(err) {
+			return nil
+		}
+		return err
+	}
+
+	err := this.runBatch(func(table string, locker *sync.Mutex) error {
+		_, err := this.Query(tx).
+			Table(table).
+			Attr("nodeId", nodeId).
+			Attr("serverId", serverId).
+			Attr("itemId", itemId).
+			Attr("time", time).
+			Delete()
+		return err
+	})
+
 	return err
 }
 
 // CountItemStats 计算统计数据数量
 func (this *MetricStatDAO) CountItemStats(tx *dbs.Tx, itemId int64, version int32) (int64, error) {
-	return this.Query(tx).
+	var total int64 = 0
-		Attr("itemId", itemId).
+
-		Attr("version", version).
+	err := this.runBatch(func(table string, locker *sync.Mutex) error {
-		Count()
+		count, err := this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Count()
+		if err != nil {
+			return err
+		}
+		atomic.AddInt64(&total, count)
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
 }
 
 // ListItemStats 列出单页统计数据
 func (this *MetricStatDAO) ListItemStats(tx *dbs.Tx, itemId int64, version int32, offset int64, size int64) (result []*MetricStat, err error) {
-	_, err = this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("itemId", itemId).
+		var partialResult = []*MetricStat{}
-		Attr("version", version).
+		_, err = this.Query(tx).
-		Offset(offset).
+			Table(table).
-		Limit(size).
+			Attr("itemId", itemId).
-		Desc("time").
+			Attr("version", version).
-		Desc("serverId").
+			Offset(offset).
-		Desc("value").
+			Limit(size).
-		Slice(&result).
+			Desc("time").
-		FindAll()
+			Desc("serverId").
+			Desc("value").
+			Slice(&partialResult).
+			FindAll()
+		if err != nil {
+			return err
+		}
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].Time > result[j].Time {
+			return true
+		}
+		if result[i].ServerId > result[j].ServerId {
+			return true
+		}
+		if result[i].Value > result[j].Value {
+			return true
+		}
+		return false
+	})
+
 	return
 }
 
@@ -139,92 +225,127 @@ func (this *MetricStatDAO) ListItemStats(tx *dbs.Tx, itemId int64, version int32
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsAtLastTime(tx *dbs.Tx, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	// 最近一次时间
-	statOne, err := this.Query(tx).
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
-		Attr("itemId", itemId).
+	if err != nil || len(lastTime) == 0 {
-		Attr("version", version).
+		return nil, err
-		DescPk().
+	}
-		Find()
+
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Attr("time", lastTime).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("MIN(time) AS time", "SUM(value) AS value", "keys").
+			Desc("value").
+			Group("keys").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+		}
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	if statOne == nil {
+	result = this.mergeStats(result)
-		return nil, nil
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Value > result[j].Value
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
-	var lastStat = statOne.(*MetricStat)
+
-	var lastTime = lastStat.Time
-	var query = this.Query(tx).
-		Attr("itemId", itemId).
-		Attr("version", version).
-		Attr("time", lastTime).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Result("MIN(time) AS time", "SUM(value) AS value", "keys").
-		Desc("value").
-		Group("keys").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
-		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
-	}
-	_, err = query.
-		FindAll()
 	return
 }
 
 // FindItemStatsWithClusterIdAndLastTime 取得集群最近一次计时前 N 个数据
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsWithClusterIdAndLastTime(tx *dbs.Tx, clusterId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	// 最近一次时间
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
-	statOne, err := this.Query(tx).
+	if err != nil || len(lastTime) == 0 {
-		Attr("itemId", itemId).
+		return nil, err
-		Attr("version", version).
+	}
-		DescPk().
+
-		Find()
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
+		var partialResult = []*MetricStat{}
+		var query = this.Query(tx).
+			Table(table).
+			UseIndex("cluster_item_time").
+			Attr("clusterId", clusterId).
+			Attr("itemId", itemId).
+			Attr("version", version).
+			Attr("time", lastTime).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("MIN(time) AS time", "SUM(value) AS value", "keys").
+			Desc("value").
+			Group("keys").
+			Limit(size).
+			Slice(&partialResult)
+		if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+		}
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
-	if statOne == nil {
-		return nil, nil
-	}
-	var lastStat = statOne.(*MetricStat)
-	var lastTime = lastStat.Time
 
-	var query = this.Query(tx).
+	result = this.mergeStats(result)
-		UseIndex("cluster_item_time").
+
-		Attr("clusterId", clusterId).
+	sort.Slice(result, func(i, j int) bool {
-		Attr("itemId", itemId).
+		return result[i].Value > result[j].Value
-		Attr("version", version).
+	})
-		Attr("time", lastTime).
+
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+	if len(result) > types.Int(size) {
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+		result = result[:types.Int(size)]
-		Result("MIN(time) AS time", "SUM(value) AS value", "keys").
-		Desc("value").
-		Group("keys").
-		Limit(size).
-		Slice(&result)
-	if ignoreEmptyKeys {
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
-		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
 	}
 
-	_, err = query.
-		FindAll()
 	return
 }
 
@@ -232,68 +353,80 @@ func (this *MetricStatDAO) FindItemStatsWithClusterIdAndLastTime(tx *dbs.Tx, clu
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsWithNodeIdAndLastTime(tx *dbs.Tx, nodeId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	// 最近一次时间
-	statOne, err := this.Query(tx).
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
-		Attr("itemId", itemId).
-		Attr("version", version).
-		DescPk().
-		Find()
 	if err != nil {
 		return nil, err
 	}
-	if statOne == nil {
+
-		return nil, nil
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-	}
+		var partialResult = []*MetricStat{}
-	var lastStat = statOne.(*MetricStat)
+		var query = this.Query(tx).
-	var lastTime = lastStat.Time
+			Table(table).
-	var query = this.Query(tx).
+			UseIndex("node_item_time").
-		UseIndex("node_item_time").
+			Attr("nodeId", nodeId).
-		Attr("nodeId", nodeId).
+			Attr("itemId", itemId).
-		Attr("itemId", itemId).
+			Attr("version", version).
-		Attr("version", version).
+			Attr("time", lastTime).
-		Attr("time", lastTime).
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Result("MIN(time) AS time", "SUM(value) AS value", "keys").
-		Result("MIN(time) AS time", "SUM(value) AS value", "keys").
+			Desc("value").
-		Desc("value").
+			Group("keys").
-		Group("keys").
+			Limit(size).
-		Limit(size).
+			Slice(&partialResult)
-		Slice(&result)
+		if ignoreEmptyKeys {
-	if ignoreEmptyKeys {
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-	}
-	if len(ignoreKeys) > 0 {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
-		if err != nil {
-			return nil, err
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+		if len(ignoreKeys) > 0 {
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Value > result[j].Value
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
 
-	_, err = query.
-		FindAll()
 	return
 }
 
-// FindItemStatsWithServerIdAndLastTime 取得节点最近一次计时前 N 个数据
+// FindItemStatsWithServerIdAndLastTime 取得服务最近一次计时前 N 个数据
 // 适合每条数据中包含不同的Key的场景
 func (this *MetricStatDAO) FindItemStatsWithServerIdAndLastTime(tx *dbs.Tx, serverId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	// 最近一次时间
-	statOne, err := this.Query(tx).
+	lastTime, err := SharedMetricItemDAO.FindMetricLastTime(tx, itemId)
-		Attr("itemId", itemId).
+	if err != nil || len(lastTime) == 0 {
-		Attr("version", version).
-		DescPk().
-		Find()
-	if err != nil {
 		return nil, err
 	}
-	if statOne == nil {
-		return nil, nil
-	}
-	var lastStat = statOne.(*MetricStat)
-	var lastTime = lastStat.Time
 
 	var query = this.Query(tx).
+		Table(this.partialTable(serverId)).
 		UseIndex("server_item_time").
 		Attr("serverId", serverId).
 		Attr("itemId", itemId).
@@ -326,68 +459,119 @@ func (this *MetricStatDAO) FindItemStatsWithServerIdAndLastTime(tx *dbs.Tx, serv
 // FindLatestItemStats 取得所有集群上最近 N 个时间的数据
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStats(tx *dbs.Tx, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	var query = this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("itemId", itemId).
+		var partialResult = []*MetricStat{}
-		Attr("version", version).
+		var query = this.Query(tx).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			Table(table).
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Attr("itemId", itemId).
-		Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
+			Attr("version", version).
-		Desc("time").
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		Group("time").
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Limit(size).
+			Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
-		Slice(&result)
+			Desc("time").
-	if ignoreEmptyKeys {
+			Group("time").
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+			Limit(size).
-	}
+			Slice(&partialResult)
-	if len(ignoreKeys) > 0 {
+		if ignoreEmptyKeys {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-		if err != nil {
+		}
-			return nil, err
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
-	}
 
-	_, err = query.
+		_, err = query.
-		FindAll()
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
 	if err != nil {
 		return nil, err
 	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Time > result[j].Time
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	lists.Reverse(result)
+
 	return
 }
 
 // FindLatestItemStatsWithClusterId 取得集群最近 N 个时间的数据
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStatsWithClusterId(tx *dbs.Tx, clusterId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	var query = this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("clusterId", clusterId).
+		var partialResult = []*MetricStat{}
-		Attr("itemId", itemId).
+		var query = this.Query(tx).
-		Attr("version", version).
+			Table(table).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			Attr("clusterId", clusterId).
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Attr("itemId", itemId).
-		Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
+			Attr("version", version).
-		Desc("time").
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		Group("time").
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Limit(size).
+			Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
-		Slice(&result)
+			Desc("time").
-	if ignoreEmptyKeys {
+			Group("time").
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+			Limit(size).
-	}
+			Slice(&partialResult)
-	if len(ignoreKeys) > 0 {
+		if ignoreEmptyKeys {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-		if err != nil {
+		}
-			return nil, err
+		if len(ignoreKeys) > 0 {
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
-	}
 
-	_, err = query.
+		_, err = query.
-		FindAll()
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
 	if err != nil {
 		return nil, err
 	}
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Time > result[j].Time
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
+	}
+
 	lists.Reverse(result)
 	return
 }
@@ -395,34 +579,55 @@ func (this *MetricStatDAO) FindLatestItemStatsWithClusterId(tx *dbs.Tx, clusterI
 // FindLatestItemStatsWithNodeId 取得节点最近 N 个时间的数据
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStatsWithNodeId(tx *dbs.Tx, nodeId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
-	var query = this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("nodeId", nodeId).
+		var partialResult = []*MetricStat{}
-		Attr("itemId", itemId).
+		var query = this.Query(tx).
-		Attr("version", version).
+			Table(table).
-		// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
+			Attr("nodeId", nodeId).
-		// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
+			Attr("itemId", itemId).
-		Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
+			Attr("version", version).
-		Desc("time").
+			// TODO 增加更多聚合算法，比如 AVG、MEDIAN、MIN、MAX 等
-		Group("time").
+			// TODO 这里的 MIN(`keys`) 在MySQL8中可以换成FIRST_VALUE
-		Limit(size).
+			Result("time", "SUM(value) AS value", "MIN(`keys`) AS `keys`").
-		Slice(&result)
+			Desc("time").
-	if ignoreEmptyKeys {
+			Group("time").
-		query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
+			Limit(size).
-	}
+			Slice(&partialResult)
-	if len(ignoreKeys) > 0 {
+		if ignoreEmptyKeys {
-		ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			query.Where("NOT JSON_CONTAINS(`keys`, '\"\"')")
-		if err != nil {
-			return nil, err
 		}
-		query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+		if len(ignoreKeys) > 0 {
-		query.Param("ignoredKeys", string(ignoreKeysJSON))
+			ignoreKeysJSON, err := json.Marshal(ignoreKeys)
+			if err != nil {
+				return err
+			}
+			query.Where("NOT JSON_CONTAINS(:ignoredKeys, JSON_EXTRACT(`keys`, '$[0]'))") // TODO $[0] 需要换成keys中的primary key位置
+			query.Param("ignoredKeys", string(ignoreKeysJSON))
+		}
+
+		_, err = query.
+			FindAll()
+		if err != nil {
+			return err
+		}
+
+		locker.Lock()
+		result = append(result, partialResult...)
+		locker.Unlock()
+
+		return nil
+	})
+
+	result = this.mergeStats(result)
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Time > result[j].Time
+	})
+
+	if len(result) > types.Int(size) {
+		result = result[:types.Int(size)]
 	}
 
-	_, err = query.
-		FindAll()
-	if err != nil {
-		return nil, err
-	}
 	lists.Reverse(result)
 	return
 }
@@ -431,6 +636,7 @@ func (this *MetricStatDAO) FindLatestItemStatsWithNodeId(tx *dbs.Tx, nodeId int6
 // 适合同个Key在不同时间段的变化场景
 func (this *MetricStatDAO) FindLatestItemStatsWithServerId(tx *dbs.Tx, serverId int64, itemId int64, ignoreEmptyKeys bool, ignoreKeys []string, version int32, size int64) (result []*MetricStat, err error) {
 	var query = this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Attr("serverId", serverId).
 		Attr("itemId", itemId).
 		Attr("version", version).
@@ -474,17 +680,21 @@ func (this *MetricStatDAO) Clean(tx *dbs.Tx) error {
 			}
 			for _, item := range items {
 				var config = &serverconfigs.MetricItemConfig{
-					Id:         int64(item.Id),
+					Id:            int64(item.Id),
-					Period:     int(item.Period),
+					Period:        int(item.Period),
-					PeriodUnit: item.PeriodUnit,
+					PeriodUnit:    item.PeriodUnit,
+					ExpiresPeriod: int(item.ExpiresPeriod),
 				}
 				var expiresDay = config.ServerExpiresDay()
-				_, err := this.Query(tx).
+				err := this.runBatch(func(table string, locker *sync.Mutex) error {
-					Attr("itemId", item.Id).
+					_, err := this.Query(tx).
-					Lte("createdDay", expiresDay).
+						Table(table).
-					UseIndex("createdDay").
+						Attr("itemId", item.Id).
-					Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作
+						Lte("createdDay", expiresDay).
-					Delete()
+						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
+						Delete()
+					return err
+				})
 				if err != nil {
 					return err
 				}
@@ -499,3 +709,60 @@ func (this *MetricStatDAO) Clean(tx *dbs.Tx) error {
 	}
 	return nil
 }
+
+// 获取分区表
+func (this *MetricStatDAO) partialTable(serverId int64) string {
+	return this.Table + "_" + types.String(serverId%int64(MetricStatTablePartials))
+}
+
+// 批量执行
+func (this *MetricStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
+	var locker = &sync.Mutex{}
+	var wg = sync.WaitGroup{}
+	wg.Add(MetricStatTablePartials)
+	var resultErr error
+	for i := 0; i < MetricStatTablePartials; i++ {
+		var table = this.partialTable(int64(i))
+		go func(table string) {
+			defer wg.Done()
+
+			err := f(table, locker)
+			if err != nil {
+				resultErr = err
+			}
+		}(table)
+	}
+	wg.Wait()
+	return resultErr
+}
+
+// 合并统计数据
+func (this *MetricStatDAO) mergeStats(stats []*MetricStat) (result []*MetricStat) {
+	var m = map[string]*MetricStat{} // key+time => *MetricStat
+	for _, stat := range stats {
+		var uniqueKey = stat.Time + "@" + string(stat.Keys)
+		oldStat, ok := m[uniqueKey]
+		if !ok {
+			result = append(result, stat)
+			m[uniqueKey] = stat
+		} else {
+			oldStat.Value += stat.Value
+		}
+	}
+	return result
+}
+
+// 检查错误是否可以忽略
+func (this *MetricStatDAO) canIgnore(err error) bool {
+	if err == nil {
+		return true
+	}
+
+	// 忽略 Error 1213: Deadlock found 错误
+	mysqlErr, ok := err.(*mysql.MySQLError)
+	if ok && mysqlErr.Number == 1213 {
+		return true
+	}
+
+	return false
+}

internal/db/models/metric_stat_dao_test.go

@@ -1,6 +1,7 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/dbs"
@@ -8,11 +9,12 @@ import (
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
 	"testing"
+	"time"
 )
 
 func TestNewMetricStatDAO_InsertMany(t *testing.T) {
 	for i := 0; i <= 1; i++ {
-		err := NewMetricStatDAO().CreateStat(nil, types.String(i)+"_v1", 18, int64(rands.Int(0, 10000)), int64(rands.Int(0, 10000)), int64(rands.Int(0, 100)), []string{"/html" + types.String(i)}, 1, timeutil.Format("Ymd"), 0)
+		err := models.NewMetricStatDAO().CreateStat(nil, types.String(i)+"_v1", 18, int64(rands.Int(0, 10000)), int64(rands.Int(0, 10000)), int64(rands.Int(0, 100)), []string{"/html" + types.String(i)}, 1, timeutil.Format("Ymd"), 0)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -23,12 +25,38 @@ func TestNewMetricStatDAO_InsertMany(t *testing.T) {
 	t.Log("done")
 }
 
-func TestMetricStatDAO_Clean(t *testing.T) {
+func TestMetricStatDAO_Clean2(t *testing.T) {
 	dbs.NotifyReady()
 
-	err := NewMetricStatDAO().Clean(nil)
+	err := models.NewMetricStatDAO().Clean(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Log("ok")
 }
+
+func TestMetricStatDAO_DeleteNodeItemStats(t *testing.T) {
+	var dao = models.NewMetricStatDAO()
+	var before = time.Now()
+	defer func() {
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}()
+	err := dao.DeleteNodeItemStats(nil, 1, 0, 1, timeutil.Format("Ymd"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok")
+}
+
+func TestMetricStatDAO_CountItemStats(t *testing.T) {
+	var dao = models.NewMetricStatDAO()
+	var before = time.Now()
+	defer func() {
+		t.Log(time.Since(before).Seconds()*1000, "ms")
+	}()
+	count, err := dao.CountItemStats(nil, 1, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("count:", count)
+}

internal/db/models/metric_sum_stat_dao.go

@@ -2,19 +2,24 @@ package models
 
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/goman"
+	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
-	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"sync"
 	"time"
 )
 
 type MetricSumStatDAO dbs.DAO
 
+const MetricSumStatTablePartials = 20 // 表格Partial数量
+
 func init() {
 	dbs.OnReadyDone(func() {
 		// 清理数据任务
@@ -23,7 +28,7 @@ func init() {
 			for range ticker.C {
 				err := SharedMetricSumStatDAO.Clean(nil)
 				if err != nil {
-					logs.Println("SharedMetricSumStatDAO: clean expired data failed: " + err.Error())
+					remotelogs.Error("SharedMetricSumStatDAO", "clean expired data failed: "+err.Error())
 				}
 			}
 		})
@@ -51,7 +56,8 @@ func init() {
 
 // UpdateSum 更新统计数据
 func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int64, serverId int64, time string, itemId int64, version int32, count int64, total float32) error {
-	return this.Query(tx).
+	err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		InsertOrUpdateQuickly(maps.Map{
 			"clusterId":  clusterId,
 			"nodeId":     nodeId,
@@ -66,11 +72,16 @@ func (this *MetricSumStatDAO) UpdateSum(tx *dbs.Tx, clusterId int64, nodeId int6
 			"count": count,
 			"total": total,
 		})
+	if this.canIgnore(err) {
+		return nil
+	}
+	return err
 }
 
 // FindNodeServerSum 查找某个服务在某个节点上的统计数据
 func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
 	one, err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		Attr("nodeId", nodeId).
 		Attr("serverId", serverId).
 		Attr("time", time).
@@ -83,29 +94,42 @@ func (this *MetricSumStatDAO) FindNodeServerSum(tx *dbs.Tx, nodeId int64, server
 	if one == nil {
 		return
 	}
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+
+	count = int64(one.(*MetricSumStat).Count)
+	total = float32(one.(*MetricSumStat).Total)
+
+	return
 }
 
 // FindSumAtTime 查找某个时间的统计数据
 func (this *MetricSumStatDAO) FindSumAtTime(tx *dbs.Tx, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("time", time).
+		one, err := this.Query(tx).
-		Attr("itemId", itemId).
+			Table(table).
-		Attr("version", version).
+			Attr("time", time).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
+			Attr("itemId", itemId).
-		Find()
+			Attr("version", version).
-	if err != nil {
+			Result("SUM(count) AS `count`, SUM(total) AS total").
-		return 0, 0, err
+			Find()
-	}
+		if err != nil {
-	if one == nil {
+			return err
-		return
+		}
-	}
+		if one == nil {
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+	return
 }
 
 // FindServerSum 查找某个服务的统计数据
 func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
 	one, err := this.Query(tx).
+		Table(this.partialTable(serverId)).
 		UseIndex("server_item_time").
 		Attr("serverId", serverId).
 		Attr("time", time).
@@ -124,48 +148,69 @@ func (this *MetricSumStatDAO) FindServerSum(tx *dbs.Tx, serverId int64, time str
 
 // FindClusterSum 查找集群上的统计数据
 func (this *MetricSumStatDAO) FindClusterSum(tx *dbs.Tx, clusterId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		UseIndex("cluster_item_time").
+		one, err := this.Query(tx).
-		Attr("clusterId", clusterId).
+			Table(table).
-		Attr("time", time).
+			UseIndex("cluster_item_time").
-		Attr("itemId", itemId).
+			Attr("clusterId", clusterId).
-		Attr("version", version).
+			Attr("time", time).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
+			Attr("itemId", itemId).
-		Find()
+			Attr("version", version).
-	if err != nil {
+			Result("SUM(count) AS `count`, SUM(total) AS total").
-		return 0, 0, err
+			Find()
-	}
+		if err != nil {
-	if one == nil {
+			return err
-		return
+		}
-	}
+		if one == nil {
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+
+	return
 }
 
 // FindNodeSum 查找节点上的统计数据
 func (this *MetricSumStatDAO) FindNodeSum(tx *dbs.Tx, nodeId int64, time string, itemId int64, version int32) (count int64, total float32, err error) {
-	one, err := this.Query(tx).
+	err = this.runBatch(func(table string, locker *sync.Mutex) error {
-		UseIndex("node_item_time").
+		one, err := this.Query(tx).
-		Attr("nodeId", nodeId).
+			Table(table).
-		Attr("time", time).
+			UseIndex("node_item_time").
-		Attr("itemId", itemId).
+			Attr("nodeId", nodeId).
-		Attr("version", version).
+			Attr("time", time).
-		Result("SUM(count) AS `count`, SUM(total) AS total").
+			Attr("itemId", itemId).
-		Find()
+			Attr("version", version).
-	if err != nil {
+			Result("SUM(count) AS `count`, SUM(total) AS total").
-		return 0, 0, err
+			Find()
-	}
+		if err != nil {
-	if one == nil {
+			return err
-		return
+		}
-	}
+		if one == nil {
-	return int64(one.(*MetricSumStat).Count), float32(one.(*MetricSumStat).Total), nil
+			return nil
+		}
+		locker.Lock()
+		count += int64(one.(*MetricSumStat).Count)
+		total += float32(one.(*MetricSumStat).Total)
+		locker.Unlock()
+		return nil
+	})
+
+	return
 }
 
 // DeleteItemStats 删除某个指标相关的统计数据
 func (this *MetricSumStatDAO) DeleteItemStats(tx *dbs.Tx, itemId int64) error {
-	_, err := this.Query(tx).
+	return this.runBatch(func(table string, locker *sync.Mutex) error {
-		Attr("itemId", itemId).
+		_, err := this.Query(tx).
-		Delete()
+			Table(table).
-	return err
+			Attr("itemId", itemId).
+			Delete()
+		return err
+	})
 }
 
 // Clean 清理数据
@@ -180,18 +225,22 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 			}
 			for _, item := range items {
 				var config = &serverconfigs.MetricItemConfig{
-					Id:         int64(item.Id),
+					Id:            int64(item.Id),
-					Period:     int(item.Period),
+					Period:        int(item.Period),
-					PeriodUnit: item.PeriodUnit,
+					PeriodUnit:    item.PeriodUnit,
+					ExpiresPeriod: int(item.ExpiresPeriod),
 				}
 				var expiresDay = config.ServerExpiresDay()
-				_, err := this.Query(tx).
+				err = this.runBatch(func(table string, locker *sync.Mutex) error {
-					Attr("itemId", item.Id).
+					_, err := this.Query(tx).
-					Where("(createdDay IS NULL OR createdDay<:day)").
+						Table(table).
-					Param("day", expiresDay).
+						Attr("itemId", item.Id).
-					UseIndex("createdDay").
+						Where("(createdDay IS NULL OR createdDay<:day)").
-					Limit(100_000). // 一次性不要删除太多，防止阻塞其他操作
+						Param("day", expiresDay).
-					Delete()
+						Limit(10_000). // 一次性不要删除太多，防止阻塞其他操作
+						Delete()
+					return err
+				})
 				if err != nil {
 					return err
 				}
@@ -206,3 +255,44 @@ func (this *MetricSumStatDAO) Clean(tx *dbs.Tx) error {
 	}
 	return nil
 }
+
+// 获取分区表
+func (this *MetricSumStatDAO) partialTable(serverId int64) string {
+	return this.Table + "_" + types.String(serverId%int64(MetricSumStatTablePartials))
+}
+
+// 批量执行
+func (this *MetricSumStatDAO) runBatch(f func(table string, locker *sync.Mutex) error) error {
+	var locker = &sync.Mutex{}
+	var wg = sync.WaitGroup{}
+	wg.Add(MetricSumStatTablePartials)
+	var resultErr error
+	for i := 0; i < MetricSumStatTablePartials; i++ {
+		var table = this.partialTable(int64(i))
+		go func(table string) {
+			defer wg.Done()
+
+			err := f(table, locker)
+			if err != nil {
+				resultErr = err
+			}
+		}(table)
+	}
+	wg.Wait()
+	return resultErr
+}
+
+// 检查错误是否可以忽略
+func (this *MetricSumStatDAO) canIgnore(err error) bool {
+	if err == nil {
+		return true
+	}
+
+	// 忽略 Error 1213: Deadlock found 错误
+	mysqlErr, ok := err.(*mysql.MySQLError)
+	if ok && mysqlErr.Number == 1213 {
+		return true
+	}
+
+	return false
+}

internal/db/models/metric_sum_stat_dao_test.go

@@ -1,16 +1,22 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
 	"github.com/iwind/TeaGo/dbs"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"testing"
 )
 
+func TestMetricSumStatDAO_FindNodeSum(t *testing.T) {
+	t.Log(models.NewMetricSumStatDAO().FindNodeSum(nil, 46, timeutil.Format("Ymd"), 1, 1))
+}
+
 func TestMetricSumStatDAO_Clean(t *testing.T) {
 	dbs.NotifyReady()
 
-	err := NewMetricSumStatDAO().Clean(nil)
+	err := models.NewMetricSumStatDAO().Clean(nil)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/db/models/monitor_node_dao.go

@@ -47,12 +47,17 @@ func (this *MonitorNodeDAO) EnableMonitorNode(tx *dbs.Tx, id int64) error {
 }
 
 // DisableMonitorNode 禁用条目
-func (this *MonitorNodeDAO) DisableMonitorNode(tx *dbs.Tx, id int64) error {
+func (this *MonitorNodeDAO) DisableMonitorNode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", MonitorNodeStateDisabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleMonitor, nodeId)
 }
 
 // FindEnabledMonitorNode 查找启用中的条目

internal/db/models/nameservers/ns_domain_dao.go

@@ -2,6 +2,7 @@ package nameservers
 
 import (
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -167,7 +168,7 @@ func (this *NSDomainDAO) CountAllEnabledDomains(tx *dbs.Tx, clusterId int64, use
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	return query.
@@ -190,7 +191,7 @@ func (this *NSDomainDAO) ListEnabledDomains(tx *dbs.Tx, clusterId int64, userId
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(NSDomainStateEnabled).

internal/db/models/nameservers/ns_record_dao.go

@@ -3,6 +3,7 @@ package nameservers
 import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
@@ -177,7 +178,7 @@ func (this *NSRecordDAO) CountAllEnabledDomainRecords(tx *dbs.Tx, domainId int64
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR value LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(routeCode) > 0 {
 		routeCodeJSON, err := json.Marshal(routeCode)
@@ -207,7 +208,7 @@ func (this *NSRecordDAO) ListEnabledRecords(tx *dbs.Tx, domainId int64, dnsType
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR value LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(routeCode) > 0 {
 		routeCodeJSON, err := json.Marshal(routeCode)

internal/db/models/nameservers/ns_route_dao.go

@@ -214,7 +214,7 @@ func (this *NSRouteDAO) FindAllEnabledRoutes(tx *dbs.Tx, clusterId int64, domain
 		State(NSRouteStateEnabled).
 		Slice(&result).
 		Desc("order").
-		DescPk()
+		AscPk()
 	if clusterId > 0 {
 		query.Attr("clusterId", clusterId)
 	} else {

internal/db/models/node_cluster_dao.go

@@ -4,10 +4,12 @@ import (
 	"encoding/json"
 	"errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -96,6 +98,7 @@ func (this *NodeClusterDAO) FindAllEnableClusters(tx *dbs.Tx) (result []*NodeClu
 	_, err = this.Query(tx).
 		State(NodeClusterStateEnabled).
 		Slice(&result).
+		Desc("isPinned").
 		Desc("order").
 		DescPk().
 		FindAll()
@@ -139,10 +142,11 @@ func (this *NodeClusterDAO) CreateCluster(tx *dbs.Tx, adminId int64, name string
 	// DNS设置
 	op.DnsDomainId = dnsDomainId
 	op.DnsName = dnsName
-	dnsConfig := &dnsconfigs.ClusterDNSConfig{
+	var dnsConfig = &dnsconfigs.ClusterDNSConfig{
 		NodesAutoSync:   true,
 		ServersAutoSync: true,
 		CNameRecords:    []string{},
+		CNameAsDomain:   true,
 		TTL:             0,
 	}
 	dnsJSON, err := json.Marshal(dnsConfig)
@@ -178,7 +182,7 @@ func (this *NodeClusterDAO) CreateCluster(tx *dbs.Tx, adminId int64, name string
 }
 
 // UpdateCluster 修改集群
-func (this *NodeClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name string, grantId int64, installDir string, timezone string, nodeMaxThreads int32, nodeTCPMaxConnections int32, autoOpenPorts bool) error {
+func (this *NodeClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name string, grantId int64, installDir string, timezone string, nodeMaxThreads int32, autoOpenPorts bool) error {
 	if clusterId <= 0 {
 		return errors.New("invalid clusterId")
 	}
@@ -193,11 +197,6 @@ func (this *NodeClusterDAO) UpdateCluster(tx *dbs.Tx, clusterId int64, name stri
 		nodeMaxThreads = 0
 	}
 	op.NodeMaxThreads = nodeMaxThreads
-
-	if nodeTCPMaxConnections < 0 {
-		nodeTCPMaxConnections = 0
-	}
-	op.NodeTCPMaxConnections = nodeTCPMaxConnections
 	op.AutoOpenPorts = autoOpenPorts
 
 	err := this.Save(tx, op)
@@ -221,7 +220,7 @@ func (this *NodeClusterDAO) CountAllEnabledClusters(tx *dbs.Tx, keyword string)
 		State(NodeClusterStateEnabled)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR dnsName like :keyword OR (dnsDomainId > 0 AND dnsDomainId IN (SELECT id FROM "+dns.SharedDNSDomainDAO.Table+" WHERE name LIKE :keyword AND state=1)))").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.Count()
 }
@@ -232,7 +231,7 @@ func (this *NodeClusterDAO) ListEnabledClusters(tx *dbs.Tx, keyword string, offs
 		State(NodeClusterStateEnabled)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR dnsName like :keyword OR (dnsDomainId > 0 AND dnsDomainId IN (SELECT id FROM "+dns.SharedDNSDomainDAO.Table+" WHERE name LIKE :keyword AND state=1)))").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		Offset(offset).
@@ -465,11 +464,33 @@ func (this *NodeClusterDAO) ExistClusterDNSName(tx *dbs.Tx, dnsName string, excl
 }
 
 // UpdateClusterDNS 修改集群DNS相关信息
-func (this *NodeClusterDAO) UpdateClusterDNS(tx *dbs.Tx, clusterId int64, dnsName string, dnsDomainId int64, nodesAutoSync bool, serversAutoSync bool, cnameRecords []string, ttl int32) error {
+func (this *NodeClusterDAO) UpdateClusterDNS(tx *dbs.Tx, clusterId int64, dnsName string, dnsDomainId int64, nodesAutoSync bool, serversAutoSync bool, cnameRecords []string, ttl int32, cnameAsDomain bool) error {
 	if clusterId <= 0 {
 		return errors.New("invalid clusterId")
 	}
-	op := NewNodeClusterOperator()
+
+	// 删除老的域名中相关记录
+	oldOne, err := this.Query(tx).
+		Pk(clusterId).
+		Result("dnsName", "dnsDomainId").
+		Find()
+	if err != nil {
+		return err
+	}
+	if oldOne == nil {
+		return nil
+	}
+
+	var oldCluster = oldOne.(*NodeCluster)
+	var oldDNSDomainId = int64(oldCluster.DnsDomainId)
+	if (oldDNSDomainId > 0 && oldDNSDomainId != dnsDomainId) || (oldCluster.DnsName != dnsName) {
+		err = dns.SharedDNSTaskDAO.CreateClusterRemoveTask(tx, clusterId, oldDNSDomainId, oldCluster.DnsName)
+		if err != nil {
+			return err
+		}
+	}
+
+	var op = NewNodeClusterOperator()
 	op.Id = clusterId
 	op.DnsName = dnsName
 	op.DnsDomainId = dnsDomainId
@@ -478,11 +499,12 @@ func (this *NodeClusterDAO) UpdateClusterDNS(tx *dbs.Tx, clusterId int64, dnsNam
 		cnameRecords = []string{}
 	}
 
-	dnsConfig := &dnsconfigs.ClusterDNSConfig{
+	var dnsConfig = &dnsconfigs.ClusterDNSConfig{
 		NodesAutoSync:   nodesAutoSync,
 		ServersAutoSync: serversAutoSync,
 		CNameRecords:    cnameRecords,
 		TTL:             ttl,
+		CNameAsDomain:   cnameAsDomain,
 	}
 	dnsJSON, err := json.Marshal(dnsConfig)
 	if err != nil {
@@ -897,7 +919,8 @@ func (this *NodeClusterDAO) FindClusterBasicInfo(tx *dbs.Tx, clusterId int64, ca
 
 	cluster, err := this.Query(tx).
 		Pk(clusterId).
-		Result("timeZone", "nodeMaxThreads", "nodeTCPMaxConnections", "cachePolicyId", "httpFirewallPolicyId", "autoOpenPorts").
+		State(NodeClusterStateEnabled).
+		Result("id", "timeZone", "nodeMaxThreads", "cachePolicyId", "httpFirewallPolicyId", "autoOpenPorts", "webp", "uam", "isOn", "ddosProtection").
 		Find()
 	if err != nil || cluster == nil {
 		return nil, err
@@ -908,6 +931,163 @@ func (this *NodeClusterDAO) FindClusterBasicInfo(tx *dbs.Tx, clusterId int64, ca
 	return cluster.(*NodeCluster), nil
 }
 
+// UpdateClusterWebPPolicy 修改WebP设置
+func (this *NodeClusterDAO) UpdateClusterWebPPolicy(tx *dbs.Tx, clusterId int64, webpPolicy *nodeconfigs.WebPImagePolicy) error {
+	if webpPolicy == nil {
+		err := this.Query(tx).
+			Pk(clusterId).
+			Set("webp", dbs.SQL("null")).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+
+		return this.NotifyUpdate(tx, clusterId)
+	}
+
+	webpPolicyJSON, err := json.Marshal(webpPolicy)
+	if err != nil {
+		return err
+	}
+	err = this.Query(tx).
+		Pk(clusterId).
+		Set("webp", webpPolicyJSON).
+		UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	return this.NotifyUpdate(tx, clusterId)
+}
+
+// FindClusterWebPPolicy 查询WebP设置
+func (this *NodeClusterDAO) FindClusterWebPPolicy(tx *dbs.Tx, clusterId int64, cacheMap *utils.CacheMap) (*nodeconfigs.WebPImagePolicy, error) {
+	var cacheKey = this.Table + ":FindClusterWebPPolicy:" + types.String(clusterId)
+	if cacheMap != nil {
+		cache, ok := cacheMap.Get(cacheKey)
+		if ok {
+			return cache.(*nodeconfigs.WebPImagePolicy), nil
+		}
+	}
+
+	webpJSON, err := this.Query(tx).
+		Pk(clusterId).
+		Result("webp").
+		FindJSONCol()
+	if err != nil {
+		return nil, err
+	}
+
+	if IsNull(webpJSON) {
+		return nodeconfigs.DefaultWebPImagePolicy, nil
+	}
+
+	var policy = &nodeconfigs.WebPImagePolicy{}
+	err = json.Unmarshal(webpJSON, policy)
+	if err != nil {
+		return nil, err
+	}
+	return policy, nil
+}
+
+// UpdateClusterUAMPolicy 修改UAM设置
+func (this *NodeClusterDAO) UpdateClusterUAMPolicy(tx *dbs.Tx, clusterId int64, uamPolicy *nodeconfigs.UAMPolicy) error {
+	if uamPolicy == nil {
+		err := this.Query(tx).
+			Pk(clusterId).
+			Set("uam", dbs.SQL("null")).
+			UpdateQuickly()
+		if err != nil {
+			return err
+		}
+
+		return this.NotifyUpdate(tx, clusterId)
+	}
+
+	uamPolicyJSON, err := json.Marshal(uamPolicy)
+	if err != nil {
+		return err
+	}
+	err = this.Query(tx).
+		Pk(clusterId).
+		Set("uam", uamPolicyJSON).
+		UpdateQuickly()
+	if err != nil {
+		return err
+	}
+
+	return this.NotifyUpdate(tx, clusterId)
+}
+
+// FindClusterUAMPolicy 查询设置
+func (this *NodeClusterDAO) FindClusterUAMPolicy(tx *dbs.Tx, clusterId int64, cacheMap *utils.CacheMap) (*nodeconfigs.UAMPolicy, error) {
+	var cacheKey = this.Table + ":FindClusterUAMPolicy:" + types.String(clusterId)
+	if cacheMap != nil {
+		cache, ok := cacheMap.Get(cacheKey)
+		if ok {
+			return cache.(*nodeconfigs.UAMPolicy), nil
+		}
+	}
+
+	uamJSON, err := this.Query(tx).
+		Pk(clusterId).
+		Result("uam").
+		FindJSONCol()
+	if err != nil {
+		return nil, err
+	}
+
+	if IsNull(uamJSON) {
+		return nodeconfigs.DefaultUAMPolicy, nil
+	}
+
+	var policy = &nodeconfigs.UAMPolicy{}
+	err = json.Unmarshal(uamJSON, policy)
+	if err != nil {
+		return nil, err
+	}
+	return policy, nil
+}
+
+// FindClusterDDoSProtection 获取集群的DDoS设置
+func (this *NodeClusterDAO) FindClusterDDoSProtection(tx *dbs.Tx, clusterId int64) (*ddosconfigs.ProtectionConfig, error) {
+	one, err := this.Query(tx).
+		Result("ddosProtection").
+		Pk(clusterId).
+		Find()
+	if one == nil || err != nil {
+		return nil, err
+	}
+
+	return one.(*NodeCluster).DecodeDDoSProtection(), nil
+}
+
+// UpdateClusterDDoSProtection 设置集群的DDOS设置
+func (this *NodeClusterDAO) UpdateClusterDDoSProtection(tx *dbs.Tx, clusterId int64, ddosProtection *ddosconfigs.ProtectionConfig) error {
+	if clusterId <= 0 {
+		return ErrNotFound
+	}
+
+	var op = NewNodeClusterOperator()
+	op.Id = clusterId
+
+	if ddosProtection == nil {
+		op.DdosProtection = "{}"
+	} else {
+		ddosProtectionJSON, err := json.Marshal(ddosProtection)
+		if err != nil {
+			return err
+		}
+		op.DdosProtection = ddosProtectionJSON
+	}
+
+	err := this.Save(tx, op)
+	if err != nil {
+		return err
+	}
+	return SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, 0, NodeTaskTypeDDosProtectionChanged)
+}
+
 // NotifyUpdate 通知更新
 func (this *NodeClusterDAO) NotifyUpdate(tx *dbs.Tx, clusterId int64) error {
 	return SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, 0, NodeTaskTypeConfigChanged)

internal/db/models/node_cluster_model.go

@@ -4,67 +4,71 @@ import "github.com/iwind/TeaGo/dbs"
 
 // NodeCluster 节点集群
 type NodeCluster struct {
-	Id                    uint32   `field:"id"`                    // ID
+	Id                   uint32   `field:"id"`                   // ID
-	AdminId               uint32   `field:"adminId"`               // 管理员ID
+	AdminId              uint32   `field:"adminId"`              // 管理员ID
-	UserId                uint32   `field:"userId"`                // 用户ID
+	UserId               uint32   `field:"userId"`               // 用户ID
-	IsOn                  bool     `field:"isOn"`                  // 是否启用
+	IsOn                 bool     `field:"isOn"`                 // 是否启用
-	Name                  string   `field:"name"`                  // 名称
+	Name                 string   `field:"name"`                 // 名称
-	UseAllAPINodes        uint8    `field:"useAllAPINodes"`        // 是否使用所有API节点
+	UseAllAPINodes       uint8    `field:"useAllAPINodes"`       // 是否使用所有API节点
-	ApiNodes              dbs.JSON `field:"apiNodes"`              // 使用的API节点
+	ApiNodes             dbs.JSON `field:"apiNodes"`             // 使用的API节点
-	InstallDir            string   `field:"installDir"`            // 安装目录
+	InstallDir           string   `field:"installDir"`           // 安装目录
-	Order                 uint32   `field:"order"`                 // 排序
+	Order                uint32   `field:"order"`                // 排序
-	CreatedAt             uint64   `field:"createdAt"`             // 创建时间
+	CreatedAt            uint64   `field:"createdAt"`            // 创建时间
-	GrantId               uint32   `field:"grantId"`               // 默认认证方式
+	GrantId              uint32   `field:"grantId"`              // 默认认证方式
-	State                 uint8    `field:"state"`                 // 状态
+	State                uint8    `field:"state"`                // 状态
-	AutoRegister          uint8    `field:"autoRegister"`          // 是否开启自动注册
+	AutoRegister         uint8    `field:"autoRegister"`         // 是否开启自动注册
-	UniqueId              string   `field:"uniqueId"`              // 唯一ID
+	UniqueId             string   `field:"uniqueId"`             // 唯一ID
-	Secret                string   `field:"secret"`                // 密钥
+	Secret               string   `field:"secret"`               // 密钥
-	HealthCheck           dbs.JSON `field:"healthCheck"`           // 健康检查
+	HealthCheck          dbs.JSON `field:"healthCheck"`          // 健康检查
-	DnsName               string   `field:"dnsName"`               // DNS名称
+	DnsName              string   `field:"dnsName"`              // DNS名称
-	DnsDomainId           uint32   `field:"dnsDomainId"`           // 域名ID
+	DnsDomainId          uint32   `field:"dnsDomainId"`          // 域名ID
-	Dns                   dbs.JSON `field:"dns"`                   // DNS配置
+	Dns                  dbs.JSON `field:"dns"`                  // DNS配置
-	Toa                   dbs.JSON `field:"toa"`                   // TOA配置
+	Toa                  dbs.JSON `field:"toa"`                  // TOA配置
-	CachePolicyId         uint32   `field:"cachePolicyId"`         // 缓存策略ID
+	CachePolicyId        uint32   `field:"cachePolicyId"`        // 缓存策略ID
-	HttpFirewallPolicyId  uint32   `field:"httpFirewallPolicyId"`  // WAF策略ID
+	HttpFirewallPolicyId uint32   `field:"httpFirewallPolicyId"` // WAF策略ID
-	AccessLog             dbs.JSON `field:"accessLog"`             // 访问日志设置
+	AccessLog            dbs.JSON `field:"accessLog"`            // 访问日志设置
-	SystemServices        dbs.JSON `field:"systemServices"`        // 系统服务设置
+	SystemServices       dbs.JSON `field:"systemServices"`       // 系统服务设置
-	TimeZone              string   `field:"timeZone"`              // 时区
+	TimeZone             string   `field:"timeZone"`             // 时区
-	NodeMaxThreads        uint32   `field:"nodeMaxThreads"`        // 节点最大线程数
+	NodeMaxThreads       uint32   `field:"nodeMaxThreads"`       // 节点最大线程数
-	NodeTCPMaxConnections uint32   `field:"nodeTCPMaxConnections"` // TCP最大连接数
+	DdosProtection       dbs.JSON `field:"ddosProtection"`       // DDOS端口
-	AutoOpenPorts         uint8    `field:"autoOpenPorts"`         // 是否自动尝试开放端口
+	AutoOpenPorts        uint8    `field:"autoOpenPorts"`        // 是否自动尝试开放端口
-	IsPinned              bool     `field:"isPinned"`              // 是否置顶
+	IsPinned             bool     `field:"isPinned"`             // 是否置顶
+	Webp                 dbs.JSON `field:"webp"`                 // WebP设置
+	Uam                  dbs.JSON `field:"uam"`                  // UAM设置
 }
 
 type NodeClusterOperator struct {
-	Id                    interface{} // ID
+	Id                   interface{} // ID
-	AdminId               interface{} // 管理员ID
+	AdminId              interface{} // 管理员ID
-	UserId                interface{} // 用户ID
+	UserId               interface{} // 用户ID
-	IsOn                  interface{} // 是否启用
+	IsOn                 interface{} // 是否启用
-	Name                  interface{} // 名称
+	Name                 interface{} // 名称
-	UseAllAPINodes        interface{} // 是否使用所有API节点
+	UseAllAPINodes       interface{} // 是否使用所有API节点
-	ApiNodes              interface{} // 使用的API节点
+	ApiNodes             interface{} // 使用的API节点
-	InstallDir            interface{} // 安装目录
+	InstallDir           interface{} // 安装目录
-	Order                 interface{} // 排序
+	Order                interface{} // 排序
-	CreatedAt             interface{} // 创建时间
+	CreatedAt            interface{} // 创建时间
-	GrantId               interface{} // 默认认证方式
+	GrantId              interface{} // 默认认证方式
-	State                 interface{} // 状态
+	State                interface{} // 状态
-	AutoRegister          interface{} // 是否开启自动注册
+	AutoRegister         interface{} // 是否开启自动注册
-	UniqueId              interface{} // 唯一ID
+	UniqueId             interface{} // 唯一ID
-	Secret                interface{} // 密钥
+	Secret               interface{} // 密钥
-	HealthCheck           interface{} // 健康检查
+	HealthCheck          interface{} // 健康检查
-	DnsName               interface{} // DNS名称
+	DnsName              interface{} // DNS名称
-	DnsDomainId           interface{} // 域名ID
+	DnsDomainId          interface{} // 域名ID
-	Dns                   interface{} // DNS配置
+	Dns                  interface{} // DNS配置
-	Toa                   interface{} // TOA配置
+	Toa                  interface{} // TOA配置
-	CachePolicyId         interface{} // 缓存策略ID
+	CachePolicyId        interface{} // 缓存策略ID
-	HttpFirewallPolicyId  interface{} // WAF策略ID
+	HttpFirewallPolicyId interface{} // WAF策略ID
-	AccessLog             interface{} // 访问日志设置
+	AccessLog            interface{} // 访问日志设置
-	SystemServices        interface{} // 系统服务设置
+	SystemServices       interface{} // 系统服务设置
-	TimeZone              interface{} // 时区
+	TimeZone             interface{} // 时区
-	NodeMaxThreads        interface{} // 节点最大线程数
+	NodeMaxThreads       interface{} // 节点最大线程数
-	NodeTCPMaxConnections interface{} // TCP最大连接数
+	DdosProtection       interface{} // DDOS端口
-	AutoOpenPorts         interface{} // 是否自动尝试开放端口
+	AutoOpenPorts        interface{} // 是否自动尝试开放端口
-	IsPinned              interface{} // 是否置顶
+	IsPinned             interface{} // 是否置顶
+	Webp                 interface{} // WebP设置
+	Uam                  interface{} // UAM设置
 }
 
 func NewNodeClusterOperator() *NodeClusterOperator {

internal/db/models/node_cluster_model_ext.go

@@ -3,6 +3,7 @@ package models
 import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeCommon/pkg/dnsconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
 )
 
 // DecodeDNSConfig 解析DNS配置
@@ -12,12 +13,38 @@ func (this *NodeCluster) DecodeDNSConfig() (*dnsconfigs.ClusterDNSConfig, error)
 		return &dnsconfigs.ClusterDNSConfig{
 			NodesAutoSync:   false,
 			ServersAutoSync: false,
+			CNameAsDomain:   true,
 		}, nil
 	}
-	dnsConfig := &dnsconfigs.ClusterDNSConfig{}
+	var dnsConfig = &dnsconfigs.ClusterDNSConfig{
+		CNameAsDomain: true,
+	}
 	err := json.Unmarshal(this.Dns, &dnsConfig)
 	if err != nil {
 		return nil, err
 	}
 	return dnsConfig, nil
 }
+
+// DecodeDDoSProtection 解析DDOS Protection设置
+func (this *NodeCluster) DecodeDDoSProtection() *ddosconfigs.ProtectionConfig {
+	if IsNull(this.DdosProtection) {
+		return nil
+	}
+
+	var result = &ddosconfigs.ProtectionConfig{}
+	err := json.Unmarshal(this.DdosProtection, &result)
+	if err != nil {
+		// ignore err
+	}
+	return result
+}
+
+// HasDDoSProtection 检查是否有DDOS设置
+func (this *NodeCluster) HasDDoSProtection() bool {
+	var config = this.DecodeDDoSProtection()
+	if config != nil {
+		return config.IsOn()
+	}
+	return false
+}

internal/db/models/node_dao.go

@@ -1,15 +1,20 @@
 package models
 
 import (
+	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils/numberutils"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils/sizes"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"github.com/TeaOSLab/EdgeCommon/pkg/systemconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -19,8 +24,10 @@ import (
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/rands"
 	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
 	"strconv"
 	"strings"
+	"time"
 )
 
 const (
@@ -93,7 +100,8 @@ func (this *NodeDAO) DisableNode(tx *dbs.Tx, nodeId int64) (err error) {
 		return err
 	}
 
-	return nil
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleNode, nodeId)
 }
 
 // FindEnabledNode 查找启用中的条目
@@ -136,6 +144,7 @@ func (this *NodeDAO) CreateNode(tx *dbs.Tx, adminId int64, name string, clusterI
 	if teaconst.MaxNodes > 0 {
 		count, err := this.Query(tx).
 			State(NodeStateEnabled).
+			Where("clusterId IN (SELECT id FROM " + SharedNodeClusterDAO.Table + " WHERE state=1)").
 			Count()
 		if err != nil {
 			return 0, err
@@ -190,7 +199,7 @@ func (this *NodeDAO) CreateNode(tx *dbs.Tx, adminId int64, name string, clusterI
 }
 
 // UpdateNode 修改节点
-func (this *NodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, clusterId int64, secondaryClusterIds []int64, groupId int64, regionId int64, isOn bool) error {
+func (this *NodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, clusterId int64, secondaryClusterIds []int64, groupId int64, regionId int64, isOn bool, level int) error {
 	if nodeId <= 0 {
 		return errors.New("invalid nodeId")
 	}
@@ -201,7 +210,16 @@ func (this *NodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, clusterId
 		return err
 	}
 
-	op := NewNodeOperator()
+	// 老的级别
+	oldLevel, err := this.Query(tx).
+		Pk(nodeId).
+		Result("level").
+		FindIntCol(0)
+	if err != nil {
+		return err
+	}
+
+	var op = NewNodeOperator()
 	op.Id = nodeId
 	op.Name = name
 	op.ClusterId = clusterId
@@ -227,6 +245,11 @@ func (this *NodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, clusterId
 	op.RegionId = regionId
 	op.LatestVersion = dbs.SQL("latestVersion+1")
 	op.IsOn = isOn
+
+	if teaconst.IsPlus {
+		op.Level = level
+	}
+
 	err = this.Save(tx, op)
 	if err != nil {
 		return err
@@ -237,7 +260,7 @@ func (this *NodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, clusterId
 		return err
 	}
 
-	// 通知老的集群更新
+	// 通知老的集群DNS更新
 	for _, oldClusterId := range oldClusterIds {
 		if oldClusterId != clusterId && !lists.ContainsInt64(secondaryClusterIds, oldClusterId) {
 			err = dns.SharedDNSTaskDAO.CreateClusterTask(tx, oldClusterId, dns.DNSTaskTypeClusterChange)
@@ -247,6 +270,14 @@ func (this *NodeDAO) UpdateNode(tx *dbs.Tx, nodeId int64, name string, clusterId
 		}
 	}
 
+	// 通知子级级别变更
+	if oldLevel > 1 || level > 1 {
+		err = this.NotifyLevelUpdate(tx, nodeId)
+		if err != nil {
+			return err
+		}
+	}
+
 	return this.NotifyDNSUpdate(tx, nodeId)
 }
 
@@ -277,11 +308,13 @@ func (this *NodeDAO) ListEnabledNodesMatch(tx *dbs.Tx,
 	keyword string,
 	groupId int64,
 	regionId int64,
+	level int32,
 	includeSecondaryNodes bool,
 	order string,
 	offset int64,
 	size int64) (result []*Node, err error) {
 	query := this.Query(tx).
+		Result(this.Table + ".*"). // must have table name for table joins below
 		State(NodeStateEnabled).
 		Offset(offset).
 		Limit(size).
@@ -290,14 +323,14 @@ func (this *NodeDAO) ListEnabledNodesMatch(tx *dbs.Tx,
 	// 集群
 	if clusterId > 0 {
 		if includeSecondaryNodes {
-			query.Where("(clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
+			query.Where("("+this.Table+".clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
 				Param("primaryClusterId", clusterId).
 				Param("primaryClusterIdString", types.String(clusterId))
 		} else {
-			query.Attr("clusterId", clusterId)
+			query.Attr(this.Table+".clusterId", clusterId)
 		}
 	} else {
-		query.Where("clusterId IN (SELECT id FROM " + SharedNodeClusterDAO.Table + " WHERE state=1)")
+		query.Where(this.Table + ".clusterId IN (SELECT id FROM " + SharedNodeClusterDAO.Table + " WHERE state=1)")
 	}
 
 	// 安装状态
@@ -323,12 +356,14 @@ func (this *NodeDAO) ListEnabledNodesMatch(tx *dbs.Tx,
 	// 关键词
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR JSON_EXTRACT(status,'$.hostname') LIKE :keyword OR id IN (SELECT nodeId FROM "+SharedNodeIPAddressDAO.Table+" WHERE ip LIKE :keyword))").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	// 分组
 	if groupId > 0 {
 		query.Attr("groupId", groupId)
+	} else if groupId < 0 {
+		query.Attr("groupId", 0)
 	}
 
 	// 区域
@@ -336,25 +371,84 @@ func (this *NodeDAO) ListEnabledNodesMatch(tx *dbs.Tx,
 		query.Attr("regionId", regionId)
 	}
 
+	// 级别
+	if level > 0 {
+		query.Attr("level", level)
+	}
+
 	// 排序
+	var minute = timeutil.FormatTime("YmdHi", time.Now().Unix()-60)
+	var nodeValueTable = SharedNodeValueDAO.Table
+	var valueItem = ""
+	var valueField = ""
+	var isAsc = false
+	var ifNullValue int64 = 0
 	switch order {
 	case "cpuAsc":
-		query.Asc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.cpuUsage'), 0), 0)")
+		valueItem = "cpu"
+		valueField = "usage"
+		isAsc = true
+		ifNullValue = 100
 	case "cpuDesc":
-		query.Desc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.cpuUsage'), 0), 0)")
+		valueItem = "cpu"
+		valueField = "usage"
+		isAsc = false
+		ifNullValue = -1
 	case "memoryAsc":
-		query.Asc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.memoryUsage'), 0), 0)")
+		valueItem = "memory"
+		valueField = "usage"
+		isAsc = true
+		ifNullValue = 100
 	case "memoryDesc":
-		query.Desc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.memoryUsage'), 0), 0)")
+		valueItem = "memory"
+		valueField = "usage"
+		isAsc = false
+		ifNullValue = -1
 	case "trafficInAsc":
-		query.Asc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.trafficInBytes'), 0), 0)")
+		valueItem = "trafficIn"
+		valueField = "total"
+		isAsc = true
+		ifNullValue = 60 * sizes.G
 	case "trafficInDesc":
-		query.Desc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.trafficInBytes'), 0), 0)")
+		valueItem = "trafficIn"
+		valueField = "total"
+		isAsc = false
+		ifNullValue = -1
 	case "trafficOutAsc":
-		query.Asc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.trafficOutBytes'), 0), 0)")
+		valueItem = "trafficOut"
+		valueField = "total"
+		isAsc = true
+		ifNullValue = sizes.G
 	case "trafficOutDesc":
-		query.Desc("IF(JSON_EXTRACT(status, '$.updatedAt')>UNIX_TIMESTAMP()-120, IFNULL(JSON_EXTRACT(status, '$.trafficOutBytes'), 0), 0)")
+		valueItem = "trafficOut"
+		valueField = "total"
+		isAsc = false
+		ifNullValue = -1
+	case "loadAsc":
+		valueItem = "load"
+		valueField = "load1m"
+		isAsc = true
+		ifNullValue = 1000
+	case "loadDesc":
+		valueItem = "load"
+		valueField = "load1m"
+		isAsc = false
+		ifNullValue = -1
+	default:
+		query.Desc("level")
 	}
+
+	if len(valueItem) > 0 {
+		query.Join(SharedNodeValueDAO, dbs.QueryJoinLeft, this.Table+".id="+nodeValueTable+".nodeId AND "+nodeValueTable+".role='"+nodeconfigs.NodeRoleNode+"' AND "+nodeValueTable+".item='"+valueItem+"' "+" AND "+nodeValueTable+".minute=:minute")
+		query.Param("minute", minute)
+		var ifNullSQL = "IFNULL(CAST(JSON_EXTRACT(value, '$." + valueField + "') AS DECIMAL (20, 6)), " + types.String(ifNullValue) + ")"
+		if isAsc {
+			query.Asc(ifNullSQL)
+		} else {
+			query.Desc(ifNullSQL)
+		}
+	}
+
 	query.DescPk()
 
 	_, err = query.FindAll()
@@ -399,6 +493,37 @@ func (this *NodeDAO) FindNodeClusterId(tx *dbs.Tx, nodeId int64) (int64, error)
 	return types.Int64(col), err
 }
 
+// FindNodeLevel 获取节点级别
+func (this *NodeDAO) FindNodeLevel(tx *dbs.Tx, nodeId int64) (int, error) {
+	level, err := this.Query(tx).
+		Pk(nodeId).
+		Result("level").
+		FindIntCol(0)
+	if err != nil {
+		return 0, err
+	}
+	if level < 1 {
+		level = 1
+	}
+	return level, nil
+}
+
+// FindNodeLevelInfo 获取节点级别相关信息
+func (this *NodeDAO) FindNodeLevelInfo(tx *dbs.Tx, nodeId int64) (*Node, error) {
+	one, err := this.Query(tx).
+		Pk(nodeId).
+		Result("id", "clusterId", "secondaryClusterIds", "groupId", "level").
+		Find()
+	if err != nil || one == nil {
+		return nil, err
+	}
+	var node = one.(*Node)
+	if node.Level < 1 {
+		node.Level = 1
+	}
+	return node, nil
+}
+
 // FindEnabledAndOnNodeClusterIds 获取节点所属所有可用而且启用的集群ID
 func (this *NodeDAO) FindEnabledAndOnNodeClusterIds(tx *dbs.Tx, nodeId int64) (result []int64, err error) {
 	one, err := this.Query(tx).
@@ -446,12 +571,12 @@ func (this *NodeDAO) FindEnabledNodeClusterIds(tx *dbs.Tx, nodeId int64) (result
 		result = append(result, clusterId)
 	}
 
-	for _, clusterId := range one.(*Node).DecodeSecondaryClusterIds() {
+	for _, secondaryClusterId := range one.(*Node).DecodeSecondaryClusterIds() {
-		if lists.ContainsInt64(result, clusterId) {
+		if lists.ContainsInt64(result, secondaryClusterId) {
 			continue
 		}
 
-		result = append(result, clusterId)
+		result = append(result, secondaryClusterId)
 	}
 	return
 }
@@ -473,6 +598,41 @@ func (this *NodeDAO) FindEnabledNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64
 	return result, nil
 }
 
+// FindEnabledNodesWithGroupIdAndLevel 查找当前分组下的某个级别的所有节点
+func (this *NodeDAO) FindEnabledNodesWithGroupIdAndLevel(tx *dbs.Tx, groupId int64, level int) (result []*Node, err error) {
+	if groupId <= 0 {
+		return
+	}
+	_, err = this.Query(tx).
+		State(NodeStateEnabled).
+		Result("id", "clusterId", "secondaryClusterIds", "uniqueId", "secret").
+		Attr("isOn", true).
+		Attr("groupId", groupId).
+		Attr("level", level).
+		Slice(&result).
+		FindAll()
+	return
+}
+
+// FindEnabledNodesWithClusterIdAndLevel 查找当前集群下的某个级别的所有节点
+func (this *NodeDAO) FindEnabledNodesWithClusterIdAndLevel(tx *dbs.Tx, clusterId int64, level int) (result []*Node, err error) {
+	if clusterId <= 0 {
+		return
+	}
+	_, err = this.Query(tx).
+		State(NodeStateEnabled).
+		Result("id", "clusterId", "secondaryClusterIds", "uniqueId", "secret").
+		Attr("isOn", true).
+		Where("(clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
+		Param("primaryClusterId", clusterId).
+		Param("primaryClusterIdString", types.String(clusterId)).
+		Attr("groupId", 0). // 需要去掉已经有分组的
+		Attr("level", level).
+		Slice(&result).
+		FindAll()
+	return
+}
+
 // FindAllNodeIdsMatch 匹配节点并返回节点ID
 func (this *NodeDAO) FindAllNodeIdsMatch(tx *dbs.Tx, clusterId int64, includeSecondaryNodes bool, isOn configutils.BoolState) (result []int64, err error) {
 	var query = this.Query(tx)
@@ -505,16 +665,51 @@ func (this *NodeDAO) FindAllNodeIdsMatch(tx *dbs.Tx, clusterId int64, includeSec
 }
 
 // FindAllEnabledNodesWithClusterId 获取一个集群的所有节点
-func (this *NodeDAO) FindAllEnabledNodesWithClusterId(tx *dbs.Tx, clusterId int64) (result []*Node, err error) {
+func (this *NodeDAO) FindAllEnabledNodesWithClusterId(tx *dbs.Tx, clusterId int64, includeSecondary bool) (result []*Node, err error) {
-	_, err = this.Query(tx).
+	var query = this.Query(tx)
+
+	if includeSecondary {
+		query.Where("(clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
+			Param("primaryClusterId", clusterId).
+			Param("primaryClusterIdString", types.String(clusterId))
+	} else {
+		query.Attr("clusterId", clusterId)
+	}
+
+	_, err = query.
 		State(NodeStateEnabled).
-		Attr("clusterId", clusterId).
 		DescPk().
 		Slice(&result).
 		FindAll()
+
 	return
 }
 
+// FindEnabledAndOnNodeIdsWithClusterId 查找某个集群下的所有启用的节点IDs
+func (this *NodeDAO) FindEnabledAndOnNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64, includeSecondary bool) ([]int64, error) {
+	var query = this.Query(tx)
+	if includeSecondary {
+		query.Where("(clusterId=:primaryClusterId OR JSON_CONTAINS(secondaryClusterIds, :primaryClusterIdString))").
+			Param("primaryClusterId", clusterId).
+			Param("primaryClusterIdString", types.String(clusterId))
+	} else {
+		query.Attr("clusterId", clusterId)
+	}
+	ones, err := query.
+		Attr("isOn", true).
+		State(NodeStateEnabled).
+		ResultPk().
+		FindAll()
+	if err != nil {
+		return nil, err
+	}
+	var result = []int64{}
+	for _, one := range ones {
+		result = append(result, int64(one.(*Node).Id))
+	}
+	return result, nil
+}
+
 // FindAllEnabledNodeIdsWithClusterId 获取一个集群的所有节点Ids
 func (this *NodeDAO) FindAllEnabledNodeIdsWithClusterId(tx *dbs.Tx, clusterId int64) (result []int64, err error) {
 	ones, err := this.Query(tx).
@@ -554,6 +749,7 @@ func (this *NodeDAO) CountAllEnabledNodesMatch(tx *dbs.Tx,
 	keyword string,
 	groupId int64,
 	regionId int64,
+	level int32,
 	includeSecondaryNodes bool) (int64, error) {
 	query := this.Query(tx)
 	query.State(NodeStateEnabled)
@@ -594,12 +790,14 @@ func (this *NodeDAO) CountAllEnabledNodesMatch(tx *dbs.Tx,
 	// 关键词
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR JSON_EXTRACT(status,'$.hostname') LIKE :keyword OR id IN (SELECT nodeId FROM "+SharedNodeIPAddressDAO.Table+" WHERE ip LIKE :keyword))").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	// 分组
 	if groupId > 0 {
 		query.Attr("groupId", groupId)
+	} else if groupId < 0 {
+		query.Attr("groupId", 0)
 	}
 
 	// 区域
@@ -607,15 +805,29 @@ func (this *NodeDAO) CountAllEnabledNodesMatch(tx *dbs.Tx,
 		query.Attr("regionId", regionId)
 	}
 
+	// 级别
+	if level > 0 {
+		query.Attr("level", level)
+	}
+
 	return query.Count()
 }
 
 // UpdateNodeStatus 更改节点状态
-func (this *NodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, statusJSON []byte) error {
+func (this *NodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, nodeStatus *nodeconfigs.NodeStatus) error {
-	_, err := this.Query(tx).
+	if nodeStatus == nil {
+		return nil
+	}
+
+	nodeStatusJSON, err := json.Marshal(nodeStatus)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.Query(tx).
 		Pk(nodeId).
 		Set("isActive", true).
-		Set("status", string(statusJSON)).
+		Set("status", nodeStatusJSON).
 		Update()
 	return err
 }
@@ -732,7 +944,11 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 		return nil, errors.New("node not found '" + strconv.FormatInt(nodeId, 10) + "'")
 	}
 
-	config := &nodeconfigs.NodeConfig{
+	if node.Level < 1 {
+		node.Level = 1
+	}
+
+	var config = &nodeconfigs.NodeConfig{
 		Id:       int64(node.Id),
 		NodeId:   node.UniqueId,
 		Secret:   node.Secret,
@@ -742,6 +958,8 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 		Name:     node.Name,
 		MaxCPU:   types.Int32(node.MaxCPU),
 		RegionId: int64(node.RegionId),
+		Level:    types.Int32(node.Level),
+		GroupId:  int64(node.GroupId),
 	}
 
 	// API节点IP
@@ -800,15 +1018,33 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 	var clusterIds = []int64{primaryClusterId}
 	clusterIds = append(clusterIds, node.DecodeSecondaryClusterIds()...)
 	var clusterIndex = 0
+	config.WebPImagePolicies = map[int64]*nodeconfigs.WebPImagePolicy{}
+	config.UAMPolicies = map[int64]*nodeconfigs.UAMPolicy{}
+	var allowIPMaps = map[string]bool{}
 	for _, clusterId := range clusterIds {
 		nodeCluster, err := SharedNodeClusterDAO.FindClusterBasicInfo(tx, clusterId, cacheMap)
 		if err != nil {
 			return nil, err
 		}
-		if nodeCluster == nil {
+		if nodeCluster == nil || !nodeCluster.IsOn {
 			continue
 		}
 
+		// 节点IP地址
+		nodeIPAddresses, err := SharedNodeIPAddressDAO.FindAllAccessibleIPAddressesWithClusterId(tx, nodeconfigs.NodeRoleNode, clusterId, cacheMap)
+		if err != nil {
+			return nil, err
+		}
+		for _, address := range nodeIPAddresses {
+			var ip = address.Ip
+			_, ok := allowIPMaps[ip]
+			if !ok {
+				allowIPMaps[ip] = true
+				config.AllowedIPs = append(config.AllowedIPs, ip)
+			}
+		}
+
+		// 防火墙
 		var httpFirewallPolicyId = int64(nodeCluster.HttpFirewallPolicyId)
 		if httpFirewallPolicyId > 0 {
 			firewallPolicy, err := SharedHTTPFirewallPolicyDAO.ComposeFirewallPolicy(tx, httpFirewallPolicyId, cacheMap)
@@ -843,10 +1079,30 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 		// 最大线程数、TCP连接数
 		if clusterIndex == 0 {
 			config.MaxThreads = int(nodeCluster.NodeMaxThreads)
-			config.TCPMaxConnections = int(nodeCluster.NodeTCPMaxConnections)
+			config.DDoSProtection = nodeCluster.DecodeDDoSProtection()
 			config.AutoOpenPorts = nodeCluster.AutoOpenPorts == 1
 		}
 
+		// webp
+		if IsNotNull(nodeCluster.Webp) {
+			var webpPolicy = &nodeconfigs.WebPImagePolicy{}
+			err = json.Unmarshal(nodeCluster.Webp, webpPolicy)
+			if err != nil {
+				return nil, err
+			}
+			config.WebPImagePolicies[clusterId] = webpPolicy
+		}
+
+		// UAM
+		if IsNotNull(nodeCluster.Uam) {
+			var uamPolicy = &nodeconfigs.UAMPolicy{}
+			err = json.Unmarshal(nodeCluster.Uam, uamPolicy)
+			if err != nil {
+				return nil, err
+			}
+			config.UAMPolicies[clusterId] = uamPolicy
+		}
+
 		clusterIndex++
 	}
 
@@ -891,6 +1147,16 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 		config.SystemServices = services
 	}
 
+	// DNS Resolver
+	if IsNotNull(node.DnsResolver) {
+		var dnsResolverConfig = nodeconfigs.DefaultDNSResolverConfig()
+		err = json.Unmarshal(node.DnsResolver, dnsResolverConfig)
+		if err != nil {
+			return nil, err
+		}
+		config.DNSResolver = dnsResolverConfig
+	}
+
 	// 防火墙动作
 	actions, err := SharedNodeClusterFirewallActionDAO.FindAllEnabledFirewallActions(tx, primaryClusterId, cacheMap)
 	if err != nil {
@@ -955,6 +1221,22 @@ func (this *NodeDAO) ComposeNodeConfig(tx *dbs.Tx, nodeId int64, cacheMap *utils
 	}
 	config.OCSPVersion = ocspVersion
 
+	// DDOS Protection
+	var ddosProtection = node.DecodeDDoSProtection()
+	if ddosProtection != nil {
+		if config.DDoSProtection == nil {
+			config.DDoSProtection = ddosProtection
+		} else {
+			config.DDoSProtection.Merge(ddosProtection)
+		}
+	}
+
+	// 初始化扩展配置
+	err = this.composeExtConfig(tx, config, clusterIds, cacheMap)
+	if err != nil {
+		return nil, err
+	}
+
 	return config, nil
 }
 
@@ -1215,6 +1497,49 @@ func (this *NodeDAO) UpdateNodeDNS(tx *dbs.Tx, nodeId int64, routes map[int64][]
 	return nil
 }
 
+// FindNodeDNSResolver 查找域名DNS Resolver
+func (this *NodeDAO) FindNodeDNSResolver(tx *dbs.Tx, nodeId int64) (*nodeconfigs.DNSResolverConfig, error) {
+	configJSON, err := this.Query(tx).
+		Pk(nodeId).
+		Result("dnsResolver").
+		FindJSONCol()
+	if err != nil {
+		return nil, err
+	}
+	if IsNull(configJSON) {
+		return nodeconfigs.DefaultDNSResolverConfig(), nil
+	}
+
+	var config = nodeconfigs.DefaultDNSResolverConfig()
+	err = json.Unmarshal(configJSON, config)
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+// UpdateNodeDNSResolver 修改域名DNS Resolver
+func (this *NodeDAO) UpdateNodeDNSResolver(tx *dbs.Tx, nodeId int64, dnsResolverConfig *nodeconfigs.DNSResolverConfig) error {
+	if nodeId <= 0 {
+		return ErrNotFound
+	}
+
+	configJSON, err := json.Marshal(dnsResolverConfig)
+	if err != nil {
+		return err
+	}
+
+	var op = NewNodeOperator()
+	op.Id = nodeId
+	op.DnsResolver = configJSON
+	err = this.Save(tx, op)
+	if err != nil {
+		return err
+	}
+
+	return this.NotifyUpdate(tx, nodeId)
+}
+
 // UpdateNodeSystem 设置系统信息
 func (this *NodeDAO) UpdateNodeSystem(tx *dbs.Tx, nodeId int64, maxCPU int32) error {
 	if nodeId <= 0 {
@@ -1491,19 +1816,146 @@ func (this *NodeDAO) TransferPrimaryClusterNodes(tx *dbs.Tx, primaryClusterId in
 	return nil
 }
 
-// NotifyUpdate 通知更新
+// FindParentNodeConfigs 查找父级节点配置
-func (this *NodeDAO) NotifyUpdate(tx *dbs.Tx, nodeId int64) error {
+func (this *NodeDAO) FindParentNodeConfigs(tx *dbs.Tx, nodeId int64, groupId int64, clusterIds []int64, level int) (result map[int64][]*nodeconfigs.ParentNodeConfig, err error) {
+	result = map[int64][]*nodeconfigs.ParentNodeConfig{} // clusterId => []*ParentNodeConfig
+
+	// 当前分组的L2
+	var parentNodes []*Node
+	if groupId > 0 {
+		parentNodes, err = this.FindEnabledNodesWithGroupIdAndLevel(tx, groupId, level+1)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// 当前集群的L2
+	if len(parentNodes) == 0 {
+		for _, clusterId := range clusterIds {
+			clusterParentNodes, err := this.FindEnabledNodesWithClusterIdAndLevel(tx, clusterId, level+1)
+			if err != nil {
+				return nil, err
+			}
+			if len(clusterParentNodes) > 0 {
+				parentNodes = append(parentNodes, clusterParentNodes...)
+			}
+		}
+	}
+
+	if len(parentNodes) > 0 {
+		for _, node := range parentNodes {
+			addrs, err := SharedNodeIPAddressDAO.FindNodeAccessAndUpIPAddresses(tx, int64(node.Id), nodeconfigs.NodeRoleNode)
+			if err != nil {
+				return nil, err
+			}
+			var addrStrings = []string{}
+			for _, addr := range addrs {
+				if addr.IsOn {
+					addrStrings = append(addrStrings, addr.DNSIP())
+				}
+			}
+
+			// 没有地址就跳过
+			if len(addrStrings) == 0 {
+				continue
+			}
+
+			var secretHash = fmt.Sprintf("%x", sha256.Sum256([]byte(node.UniqueId+"@"+node.Secret)))
+
+			for _, clusterId := range node.AllClusterIds() {
+				parentNodeConfigs, _ := result[clusterId]
+				parentNodeConfigs = append(parentNodeConfigs, &nodeconfigs.ParentNodeConfig{
+					Id:         int64(node.Id),
+					Addrs:      addrStrings,
+					SecretHash: secretHash,
+				})
+				result[clusterId] = parentNodeConfigs
+			}
+		}
+	}
+
+	return
+}
+
+// FindNodeDDoSProtection 获取节点的DDOS设置
+func (this *NodeDAO) FindNodeDDoSProtection(tx *dbs.Tx, nodeId int64) (*ddosconfigs.ProtectionConfig, error) {
+	one, err := this.Query(tx).
+		Result("ddosProtection").
+		Pk(nodeId).
+		Find()
+	if one == nil || err != nil {
+		return nil, err
+	}
+
+	return one.(*Node).DecodeDDoSProtection(), nil
+}
+
+// UpdateNodeDDoSProtection 设置集群的DDOS设置
+func (this *NodeDAO) UpdateNodeDDoSProtection(tx *dbs.Tx, nodeId int64, ddosProtection *ddosconfigs.ProtectionConfig) error {
+	if nodeId <= 0 {
+		return ErrNotFound
+	}
+
+	var op = NewNodeOperator()
+	op.Id = nodeId
+
+	if ddosProtection == nil {
+		op.DdosProtection = "{}"
+	} else {
+		ddosProtectionJSON, err := json.Marshal(ddosProtection)
+		if err != nil {
+			return err
+		}
+		op.DdosProtection = ddosProtectionJSON
+	}
+
+	err := this.Save(tx, op)
+	if err != nil {
+		return err
+	}
+
 	clusterId, err := this.FindNodeClusterId(tx, nodeId)
 	if err != nil {
 		return err
 	}
 	if clusterId > 0 {
-		return SharedNodeTaskDAO.CreateNodeTask(tx, nodeconfigs.NodeRoleNode, clusterId, nodeId, 0, NodeTaskTypeConfigChanged, 0)
+		return SharedNodeTaskDAO.CreateNodeTask(tx, nodeconfigs.NodeRoleNode, clusterId, nodeId, 0, NodeTaskTypeDDosProtectionChanged, 0)
 	}
 	return nil
 }
 
-// NotifyDNSUpdate 通知DNS更新
+// NotifyUpdate 通知节点相关更新
+func (this *NodeDAO) NotifyUpdate(tx *dbs.Tx, nodeId int64) error {
+	// 这里只需要通知单个集群即可，因为节点是公用的，更新一个就相当于更新了所有
+	clusterId, err := this.FindNodeClusterId(tx, nodeId)
+	if err != nil {
+		return err
+	}
+	err = SharedNodeTaskDAO.CreateNodeTask(tx, nodeconfigs.NodeRoleNode, clusterId, nodeId, 0, NodeTaskTypeConfigChanged, 0)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NotifyLevelUpdate 通知节点级别更新
+func (this *NodeDAO) NotifyLevelUpdate(tx *dbs.Tx, nodeId int64) error {
+	// 这里只需要通知单个集群即可，因为节点是公用的，更新一个就相当于更新了所有
+	clusterIds, err := this.FindEnabledAndOnNodeClusterIds(tx, nodeId)
+	if err != nil {
+		return err
+	}
+	for _, clusterId := range clusterIds {
+		err = SharedNodeTaskDAO.CreateClusterTask(tx, nodeconfigs.NodeRoleNode, clusterId, 0, NodeTaskTypeNodeLevelChanged)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// NotifyDNSUpdate 通知节点相关DNS更新
 func (this *NodeDAO) NotifyDNSUpdate(tx *dbs.Tx, nodeId int64) error {
 	clusterIds, err := this.FindEnabledAndOnNodeClusterIds(tx, nodeId)
 	if err != nil {

internal/db/models/node_dao_ext.go

@@ -0,0 +1,15 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
+//go:build !plus
+// +build !plus
+
+package models
+
+import (
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
+	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/iwind/TeaGo/dbs"
+)
+
+func (this *NodeDAO) composeExtConfig(tx *dbs.Tx, config *nodeconfigs.NodeConfig, clusterIds []int64, cacheMap *utils.CacheMap) error {
+	return nil
+}

internal/db/models/node_dao_test.go

@@ -1,16 +1,23 @@
-package models
+//go:build plus
+// +build plus
+
+package models_test
 
 import (
+	teaconst "github.com/TeaOSLab/EdgeAPI/internal/const"
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/logs"
 	"testing"
 	"time"
 )
 
 func TestNodeDAO_FindAllNodeIdsMatch(t *testing.T) {
 	var tx *dbs.Tx
-	nodeIds, err := SharedNodeDAO.FindAllNodeIdsMatch(tx, 1, true, 0)
+	dbs.NotifyReady()
+	nodeIds, err := models.SharedNodeDAO.FindAllNodeIdsMatch(tx, 1, true, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -20,7 +27,7 @@ func TestNodeDAO_FindAllNodeIdsMatch(t *testing.T) {
 func TestNodeDAO_UpdateNodeUp(t *testing.T) {
 	dbs.NotifyReady()
 	var tx *dbs.Tx
-	err := SharedNodeDAO.UpdateNodeUp(tx, 57, false)
+	err := models.SharedNodeDAO.UpdateNodeUp(tx, 57, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -30,7 +37,7 @@ func TestNodeDAO_UpdateNodeUp(t *testing.T) {
 func TestNodeDAO_FindEnabledNodeClusterIds(t *testing.T) {
 	dbs.NotifyReady()
 	var tx *dbs.Tx
-	clusterIds, err := NewNodeDAO().FindEnabledAndOnNodeClusterIds(tx, 48)
+	clusterIds, err := models.NewNodeDAO().FindEnabledAndOnNodeClusterIds(tx, 48)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -47,7 +54,7 @@ func TestNodeDAO_ComposeNodeConfig(t *testing.T) {
 
 	var tx *dbs.Tx
 	var cacheMap = utils.NewCacheMap()
-	nodeConfig, err := SharedNodeDAO.ComposeNodeConfig(tx, 48, cacheMap)
+	nodeConfig, err := models.SharedNodeDAO.ComposeNodeConfig(tx, 48, cacheMap)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -56,3 +63,17 @@ func TestNodeDAO_ComposeNodeConfig(t *testing.T) {
 
 	// old: 77ms => new: 56ms
 }
+
+func TestNodeDAO_ComposeNodeConfig_ParentNodes(t *testing.T) {
+	dbs.NotifyReady()
+
+	teaconst.IsPlus = true
+
+	var tx *dbs.Tx
+	var cacheMap = utils.NewCacheMap()
+	nodeConfig, err := models.SharedNodeDAO.ComposeNodeConfig(tx, 48, cacheMap)
+	if err != nil {
+		t.Fatal(err)
+	}
+	logs.PrintAsJSON(nodeConfig.ParentNodes, t)
+}

internal/db/models/node_grant_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"errors"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/iwind/TeaGo/Tea"
 	"github.com/iwind/TeaGo/dbs"
@@ -129,7 +130,7 @@ func (this *NodeGrantDAO) CountAllEnabledGrants(tx *dbs.Tx, keyword string) (int
 		State(NodeGrantStateEnabled)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR username LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.Count()
 }
@@ -140,7 +141,7 @@ func (this *NodeGrantDAO) ListEnabledGrants(tx *dbs.Tx, keyword string, offset i
 		State(NodeGrantStateEnabled)
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR username LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		Offset(offset).

internal/db/models/node_ip_address_dao.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/db/models/dns"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
+	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -124,7 +126,7 @@ func (this *NodeIPAddressDAO) CreateAddress(tx *dbs.Tx, adminId int64, nodeId in
 		role = nodeconfigs.NodeRoleNode
 	}
 
-	op := NewNodeIPAddressOperator()
+	var op = NewNodeIPAddressOperator()
 	op.NodeId = nodeId
 	op.Role = role
 	op.Name = name
@@ -233,17 +235,20 @@ func (this *NodeIPAddressDAO) FindAllEnabledAddressesWithNode(tx *dbs.Tx, nodeId
 }
 
 // FindFirstNodeAccessIPAddress 查找节点的第一个可访问的IP地址
-func (this *NodeIPAddressDAO) FindFirstNodeAccessIPAddress(tx *dbs.Tx, nodeId int64, role nodeconfigs.NodeRole) (ip string, addrId int64, err error) {
+func (this *NodeIPAddressDAO) FindFirstNodeAccessIPAddress(tx *dbs.Tx, nodeId int64, mustUp bool, role nodeconfigs.NodeRole) (ip string, addrId int64, err error) {
 	if len(role) == 0 {
 		role = nodeconfigs.NodeRoleNode
 	}
-	one, err := this.Query(tx).
+	var query = this.Query(tx).
 		Attr("nodeId", nodeId).
 		Attr("role", role).
 		State(NodeIPAddressStateEnabled).
 		Attr("canAccess", true).
-		Attr("isOn", true).
+		Attr("isOn", true)
-		Attr("isUp", true).
+	if mustUp {
+		query.Attr("isUp", true)
+	}
+	one, err := query.
 		Desc("order").
 		AscPk().
 		Result("id", "ip").
@@ -260,17 +265,20 @@ func (this *NodeIPAddressDAO) FindFirstNodeAccessIPAddress(tx *dbs.Tx, nodeId in
 }
 
 // FindFirstNodeAccessIPAddressId 查找节点的第一个可访问的IP地址ID
-func (this *NodeIPAddressDAO) FindFirstNodeAccessIPAddressId(tx *dbs.Tx, nodeId int64, role nodeconfigs.NodeRole) (int64, error) {
+func (this *NodeIPAddressDAO) FindFirstNodeAccessIPAddressId(tx *dbs.Tx, nodeId int64, mustUp bool, role nodeconfigs.NodeRole) (int64, error) {
 	if len(role) == 0 {
 		role = nodeconfigs.NodeRoleNode
 	}
-	return this.Query(tx).
+	var query = this.Query(tx).
 		Attr("nodeId", nodeId).
 		Attr("role", role).
 		State(NodeIPAddressStateEnabled).
 		Attr("canAccess", true).
-		Attr("isOn", true).
+		Attr("isOn", true)
-		Attr("isUp", true).
+	if mustUp {
+		query.Attr("isUp", true)
+	}
+	return query.
 		Desc("order").
 		AscPk().
 		Result("id").
@@ -323,7 +331,7 @@ func (this *NodeIPAddressDAO) CountAllEnabledIPAddresses(tx *dbs.Tx, role string
 	// 关键词
 	if len(keyword) > 0 {
 		query.Where("(ip LIKE :keyword OR name LIKE :keyword OR description LIKE :keyword OR nodeId IN (SELECT id FROM "+SharedNodeDAO.Table+" WHERE state=1 AND name LIKE :keyword))").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	return query.Count()
@@ -355,7 +363,7 @@ func (this *NodeIPAddressDAO) ListEnabledIPAddresses(tx *dbs.Tx, role string, no
 	// 关键词
 	if len(keyword) > 0 {
 		query.Where("(ip LIKE :keyword OR name LIKE :keyword OR description LIKE :keyword OR nodeId IN (SELECT id FROM "+SharedNodeDAO.Table+" WHERE state=1 AND name LIKE :keyword))").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	_, err = query.Offset(offset).
@@ -368,7 +376,15 @@ func (this *NodeIPAddressDAO) ListEnabledIPAddresses(tx *dbs.Tx, role string, no
 }
 
 // FindAllAccessibleIPAddressesWithClusterId 列出所有的正在启用的IP地址
-func (this *NodeIPAddressDAO) FindAllAccessibleIPAddressesWithClusterId(tx *dbs.Tx, role string, clusterId int64) (result []*NodeIPAddress, err error) {
+func (this *NodeIPAddressDAO) FindAllAccessibleIPAddressesWithClusterId(tx *dbs.Tx, role string, clusterId int64, cacheMap *utils.CacheMap) (result []*NodeIPAddress, err error) {
+	var cacheKey = this.Table + ":FindAllAccessibleIPAddressesWithClusterId:" + role + ":" + types.String(clusterId)
+	if cacheMap != nil {
+		cache, ok := cacheMap.Get(cacheKey)
+		if ok {
+			return cache.([]*NodeIPAddress), nil
+		}
+	}
+
 	_, err = this.Query(tx).
 		State(NodeIPAddressStateEnabled).
 		Attr("role", role).
@@ -378,6 +394,14 @@ func (this *NodeIPAddressDAO) FindAllAccessibleIPAddressesWithClusterId(tx *dbs.
 		Param("clusterId", clusterId).
 		Slice(&result).
 		FindAll()
+	if err != nil {
+		return
+	}
+
+	if cacheMap != nil {
+		cacheMap.Put(cacheKey, result)
+	}
+
 	return
 }
 
@@ -455,13 +479,13 @@ func (this *NodeIPAddressDAO) UpdateAddressBackupIP(tx *dbs.Tx, addressId int64,
 }
 
 // UpdateAddressHealthCount 计算IP健康状态
-func (this *NodeIPAddressDAO) UpdateAddressHealthCount(tx *dbs.Tx, addrId int64, isUp bool, maxUp int, maxDown int) (changed bool, err error) {
+func (this *NodeIPAddressDAO) UpdateAddressHealthCount(tx *dbs.Tx, addrId int64, newIsUp bool, maxUp int, maxDown int, autoUpDown bool) (changed bool, err error) {
 	if addrId <= 0 {
 		return false, errors.New("invalid address id")
 	}
 	one, err := this.Query(tx).
 		Pk(addrId).
-		Result("isHealthy", "countUp", "countDown").
+		Result("isHealthy", "isUp", "countUp", "countDown").
 		Find()
 	if err != nil {
 		return false, err
@@ -469,26 +493,57 @@ func (this *NodeIPAddressDAO) UpdateAddressHealthCount(tx *dbs.Tx, addrId int64,
 	if one == nil {
 		return false, nil
 	}
-	oldIsHealthy := one.(*NodeIPAddress).IsHealthy
+	var oldIsHealthy = one.(*NodeIPAddress).IsHealthy
+	var oldIsUp = one.(*NodeIPAddress).IsUp
 
 	// 如果新老状态一致，则不做任何事情
-	if oldIsHealthy == isUp {
+	if oldIsHealthy == newIsUp {
+		// 如果自动上下线，则健康状况和是否在线保持一致
+		if autoUpDown {
+			if oldIsUp != oldIsHealthy {
+				err = this.Query(tx).
+					Pk(addrId).
+					Set("isUp", newIsUp).
+					UpdateQuickly()
+				if err != nil {
+					return false, err
+				}
+				err = this.NotifyUpdate(tx, addrId)
+				if err != nil {
+					return false, err
+				}
+
+				// 创建日志
+				if newIsUp {
+					err = SharedNodeIPAddressLogDAO.CreateLog(tx, 0, addrId, "健康检查上线")
+				} else {
+					err = SharedNodeIPAddressLogDAO.CreateLog(tx, 0, addrId, "健康检查下线")
+				}
+				if err != nil {
+					return true, err
+				}
+
+				return true, nil
+			}
+		}
 		return false, nil
 	}
 
-	countUp := int(one.(*NodeIPAddress).CountUp)
+	var countUp = int(one.(*NodeIPAddress).CountUp)
-	countDown := int(one.(*NodeIPAddress).CountDown)
+	var countDown = int(one.(*NodeIPAddress).CountDown)
 
-	op := NewNodeIPAddressOperator()
+	var op = NewNodeIPAddressOperator()
 	op.Id = addrId
 
-	if isUp {
+	if newIsUp {
 		countUp++
 		countDown = 0
 
 		if countUp >= maxUp {
 			changed = true
-			//op.IsUp = true
+			if autoUpDown {
+				op.IsUp = true
+			}
 			op.IsHealthy = true
 		}
 	} else {
@@ -497,7 +552,9 @@ func (this *NodeIPAddressDAO) UpdateAddressHealthCount(tx *dbs.Tx, addrId int64,
 
 		if countDown >= maxDown {
 			changed = true
-			//op.IsUp = false
+			if autoUpDown {
+				op.IsUp = false
+			}
 			op.IsHealthy = false
 		}
 	}
@@ -514,6 +571,15 @@ func (this *NodeIPAddressDAO) UpdateAddressHealthCount(tx *dbs.Tx, addrId int64,
 		if err != nil {
 			return true, err
 		}
+
+		// 创建日志
+		if autoUpDown {
+			if newIsUp {
+				err = SharedNodeIPAddressLogDAO.CreateLog(tx, 0, addrId, "健康检查上线")
+			} else {
+				err = SharedNodeIPAddressLogDAO.CreateLog(tx, 0, addrId, "健康检查下线")
+			}
+		}
 	}
 
 	return
@@ -539,9 +605,21 @@ func (this *NodeIPAddressDAO) NotifyUpdate(tx *dbs.Tx, addressId int64) error {
 	switch role {
 	case nodeconfigs.NodeRoleNode:
 		err = dns.SharedDNSTaskDAO.CreateNodeTask(tx, nodeId, dns.DNSTaskTypeNodeChange)
-	}
+		if err != nil {
-	if err != nil {
+			return err
-		return err
+		}
+
+		// 检查是否为L2以上级别
+		level, err := SharedNodeDAO.FindNodeLevel(tx, nodeId)
+		if err != nil {
+			return err
+		}
+		if level > 1 {
+			err = SharedNodeDAO.NotifyLevelUpdate(tx, nodeId)
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }

internal/db/models/node_ip_address_dao_test.go

@@ -56,7 +56,7 @@ func TestNodeIPAddressDAO_UpdateAddressHealthCount(t *testing.T) {
 	dbs.NotifyReady()
 
 	var tx *dbs.Tx
-	isChanged, err := SharedNodeIPAddressDAO.UpdateAddressHealthCount(tx, 1, true, 3, 3)
+	isChanged, err := SharedNodeIPAddressDAO.UpdateAddressHealthCount(tx, 1, true, 3, 3, false)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/db/models/node_ip_address_log_dao_test.go

@@ -9,6 +9,6 @@ import (
 
 func TestNodeIPAddressDAO_FindFirstNodeAccessIPAddress(t *testing.T) {
 	var dao = NewNodeIPAddressDAO()
-	t.Log(dao.FindFirstNodeAccessIPAddress(nil, 48, nodeconfigs.NodeRoleNode))
+	t.Log(dao.FindFirstNodeAccessIPAddress(nil, 48, true, nodeconfigs.NodeRoleNode))
-	t.Log(dao.FindFirstNodeAccessIPAddressId(nil, 48, nodeconfigs.NodeRoleNode))
+	t.Log(dao.FindFirstNodeAccessIPAddressId(nil, 48, true, nodeconfigs.NodeRoleNode))
 }

internal/db/models/node_log_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
@@ -59,7 +60,7 @@ func (this *NodeLogDAO) CreateLog(tx *dbs.Tx, nodeRole nodeconfigs.NodeRole, nod
 		}
 	}
 
-	hash := stringutil.Md5(nodeRole + "@" + types.String(nodeId) + "@" + types.String(serverId) + "@" + types.String(originId) + "@" + level + "@" + tag + "@" + description + "@" + string(paramsJSON))
+	var hash = stringutil.Md5(nodeRole + "@" + types.String(nodeId) + "@" + types.String(serverId) + "@" + types.String(originId) + "@" + level + "@" + tag + "@" + description + "@" + string(paramsJSON))
 
 	// 检查是否在重复最后一条，避免重复创建
 	lastLog, err := this.Query(tx).
@@ -80,7 +81,7 @@ func (this *NodeLogDAO) CreateLog(tx *dbs.Tx, nodeRole nodeconfigs.NodeRole, nod
 		}
 	}
 
-	op := NewNodeLogOperator()
+	var op = NewNodeLogOperator()
 	op.Role = nodeRole
 	op.NodeId = nodeId
 	op.ServerId = serverId
@@ -139,13 +140,15 @@ func (this *NodeLogDAO) CountNodeLogs(tx *dbs.Tx,
 	nodeId int64,
 	serverId int64,
 	originId int64,
+	allServers bool,
 	dayFrom string,
 	dayTo string,
 	keyword string,
 	level string,
+	fixedState configutils.BoolState,
 	isUnread bool,
 	tag string) (int64, error) {
-	query := this.Query(tx)
+	var query = this.Query(tx)
 	if len(role) > 0 {
 		query.Attr("role", role)
 	}
@@ -166,6 +169,8 @@ func (this *NodeLogDAO) CountNodeLogs(tx *dbs.Tx,
 	}
 	if serverId > 0 {
 		query.Attr("serverId", serverId)
+	} else if allServers {
+		query.Where("serverId>0")
 	}
 	if originId > 0 {
 		query.Attr("originId", originId)
@@ -180,16 +185,23 @@ func (this *NodeLogDAO) CountNodeLogs(tx *dbs.Tx,
 	}
 	if len(keyword) > 0 {
 		query.Where("(tag LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(level) > 0 {
 		query.Attr("level", level)
 	}
+	if fixedState == configutils.BoolStateYes {
+		query.Attr("isFixed", 1)
+		query.Where("level IN ('error', 'success', 'warning')")
+	} else if fixedState == configutils.BoolStateNo {
+		query.Attr("isFixed", 0)
+		query.Where("level IN ('error', 'success', 'warning')")
+	}
 	if isUnread {
 		query.Attr("isRead", 0)
 	}
 	if len(tag) > 0 {
-		query.Like("tag", "%"+tag+"%")
+		query.Like("tag", dbutils.QuoteLikeKeyword(tag))
 	}
 
 	return query.Count()
@@ -212,7 +224,7 @@ func (this *NodeLogDAO) ListNodeLogs(tx *dbs.Tx,
 	tag string,
 	offset int64,
 	size int64) (result []*NodeLog, err error) {
-	query := this.Query(tx)
+	var query = this.Query(tx)
 	if len(role) > 0 {
 		query.Attr("role", role)
 	}
@@ -241,8 +253,10 @@ func (this *NodeLogDAO) ListNodeLogs(tx *dbs.Tx,
 	}
 	if fixedState == configutils.BoolStateYes {
 		query.Attr("isFixed", 1)
+		query.Where("level IN ('error', 'success', 'warning')")
 	} else if fixedState == configutils.BoolStateNo {
 		query.Attr("isFixed", 0)
+		query.Where("level IN ('error', 'success', 'warning')")
 	}
 	if len(dayFrom) > 0 {
 		dayFrom = strings.ReplaceAll(dayFrom, "-", "")
@@ -254,7 +268,7 @@ func (this *NodeLogDAO) ListNodeLogs(tx *dbs.Tx,
 	}
 	if len(keyword) > 0 {
 		query.Where("(tag LIKE :keyword OR description LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	if len(level) > 0 {
 		var pieces = strings.Split(level, ",")
@@ -268,7 +282,7 @@ func (this *NodeLogDAO) ListNodeLogs(tx *dbs.Tx,
 		query.Attr("isRead", 0)
 	}
 	if len(tag) > 0 {
-		query.Like("tag", "%"+tag+"%")
+		query.Like("tag", dbutils.QuoteLikeKeyword(tag))
 	}
 	_, err = query.
 		Offset(offset).
@@ -310,6 +324,14 @@ func (this *NodeLogDAO) UpdateNodeLogFixed(tx *dbs.Tx, logId int64) error {
 	return nil
 }
 
+// UpdateAllNodeLogsFixed 设置所有节点日志为已修复
+func (this *NodeLogDAO) UpdateAllNodeLogsFixed(tx *dbs.Tx) error {
+	return this.Query(tx).
+		Attr("isFixed", 0).
+		Set("isFixed", 1).
+		UpdateQuickly()
+}
+
 // CountAllUnreadNodeLogs 计算未读的日志数量
 func (this *NodeLogDAO) CountAllUnreadNodeLogs(tx *dbs.Tx) (int64, error) {
 	return this.Query(tx).
@@ -338,3 +360,12 @@ func (this *NodeLogDAO) UpdateAllNodeLogsRead(tx *dbs.Tx) error {
 		Set("isRead", true).
 		UpdateQuickly()
 }
+
+// DeleteNodeLogs 删除某个节点上的日志
+func (this *NodeLogDAO) DeleteNodeLogs(tx *dbs.Tx, role nodeconfigs.NodeRole, nodeId int64) error {
+	_, err := this.Query(tx).
+		Attr("nodeId", nodeId).
+		Attr("role", role).
+		Delete()
+	return err
+}

internal/db/models/node_model.go

@@ -7,6 +7,7 @@ type Node struct {
 	Id                     uint32   `field:"id"`                     // ID
 	AdminId                uint32   `field:"adminId"`                // 管理员ID
 	UserId                 uint32   `field:"userId"`                 // 用户ID
+	Level                  uint8    `field:"level"`                  // 级别
 	IsOn                   bool     `field:"isOn"`                   // 是否启用
 	IsUp                   bool     `field:"isUp"`                   // 是否在线
 	CountUp                uint32   `field:"countUp"`                // 连续在线次数
@@ -30,16 +31,20 @@ type Node struct {
 	State                  uint8    `field:"state"`                  // 状态
 	ConnectedAPINodes      dbs.JSON `field:"connectedAPINodes"`      // 当前连接的API节点
 	MaxCPU                 uint32   `field:"maxCPU"`                 // 可以使用的最多CPU
+	MaxThreads             uint32   `field:"maxThreads"`             // 最大线程数
+	DdosProtection         dbs.JSON `field:"ddosProtection"`         // DDOS配置
 	DnsRoutes              dbs.JSON `field:"dnsRoutes"`              // DNS线路设置
 	MaxCacheDiskCapacity   dbs.JSON `field:"maxCacheDiskCapacity"`   // 硬盘缓存容量
 	MaxCacheMemoryCapacity dbs.JSON `field:"maxCacheMemoryCapacity"` // 内存缓存容量
 	CacheDiskDir           string   `field:"cacheDiskDir"`           // 缓存目录
+	DnsResolver            dbs.JSON `field:"dnsResolver"`            // DNS解析器
 }
 
 type NodeOperator struct {
 	Id                     interface{} // ID
 	AdminId                interface{} // 管理员ID
 	UserId                 interface{} // 用户ID
+	Level                  interface{} // 级别
 	IsOn                   interface{} // 是否启用
 	IsUp                   interface{} // 是否在线
 	CountUp                interface{} // 连续在线次数
@@ -63,10 +68,13 @@ type NodeOperator struct {
 	State                  interface{} // 状态
 	ConnectedAPINodes      interface{} // 当前连接的API节点
 	MaxCPU                 interface{} // 可以使用的最多CPU
+	MaxThreads             interface{} // 最大线程数
+	DdosProtection         interface{} // DDOS配置
 	DnsRoutes              interface{} // DNS线路设置
 	MaxCacheDiskCapacity   interface{} // 硬盘缓存容量
 	MaxCacheMemoryCapacity interface{} // 内存缓存容量
 	CacheDiskDir           interface{} // 缓存目录
+	DnsResolver            interface{} // DNS解析器
 }
 
 func NewNodeOperator() *NodeOperator {

internal/db/models/node_model_ext.go

@@ -3,6 +3,8 @@ package models
 import (
 	"encoding/json"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/ddosconfigs"
+	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
 	"sort"
 	"time"
 )
@@ -96,3 +98,73 @@ func (this *Node) DecodeSecondaryClusterIds() []int64 {
 	_ = json.Unmarshal(this.SecondaryClusterIds, &result)
 	return result
 }
+
+// AllClusterIds 获取所属集群IDs
+func (this *Node) AllClusterIds() []int64 {
+	var result = []int64{}
+
+	if this.ClusterId > 0 {
+		result = append(result, int64(this.ClusterId))
+	}
+
+	result = append(result, this.DecodeSecondaryClusterIds()...)
+
+	return result
+}
+
+// DecodeDDoSProtection 解析DDoS Protection设置
+func (this *Node) DecodeDDoSProtection() *ddosconfigs.ProtectionConfig {
+	if IsNull(this.DdosProtection) {
+		return nil
+	}
+
+	var result = &ddosconfigs.ProtectionConfig{}
+	err := json.Unmarshal(this.DdosProtection, &result)
+	if err != nil {
+		// ignore err
+	}
+	return result
+}
+
+// HasDDoSProtection 检查是否有DDOS设置
+func (this *Node) HasDDoSProtection() bool {
+	var config = this.DecodeDDoSProtection()
+	if config != nil {
+		return !config.IsPriorEmpty()
+	}
+	return false
+}
+
+func (this *Node) DecodeMaxCacheDiskCapacity() *shared.SizeCapacity {
+	if this.MaxCacheDiskCapacity.IsNull() {
+		return nil
+	}
+
+	// ignore error
+	capacity, _ := shared.DecodeSizeCapacityJSON(this.MaxCacheDiskCapacity)
+	return capacity
+}
+
+func (this *Node) DecodeMaxCacheMemoryCapacity() *shared.SizeCapacity {
+	if this.MaxCacheMemoryCapacity.IsNull() {
+		return nil
+	}
+
+	// ignore error
+	capacity, _ := shared.DecodeSizeCapacityJSON(this.MaxCacheMemoryCapacity)
+	return capacity
+}
+
+// DecodeDNSResolver 解析DNS解析主机配置
+func (this *Node) DecodeDNSResolver() *nodeconfigs.DNSResolverConfig {
+	if this.DnsResolver.IsNull() {
+		return nil
+	}
+
+	var resolverConfig = nodeconfigs.DefaultDNSResolverConfig()
+	err := json.Unmarshal(this.DnsResolver, resolverConfig)
+	if err != nil {
+		// ignore error
+	}
+	return resolverConfig
+}

internal/db/models/node_task_dao.go

@@ -14,9 +14,12 @@ import (
 type NodeTaskType = string
 
 const (
-	NodeTaskTypeConfigChanged      NodeTaskType = "configChanged"
+	NodeTaskTypeConfigChanged         NodeTaskType = "configChanged"         // 节点整体配置变化
-	NodeTaskTypeIPItemChanged      NodeTaskType = "ipItemChanged"
+	NodeTaskTypeDDosProtectionChanged NodeTaskType = "ddosProtectionChanged" // 节点DDoS配置变更
-	NodeTaskTypeNodeVersionChanged NodeTaskType = "nodeVersionChanged"
+	NodeTaskTypeIPItemChanged         NodeTaskType = "ipItemChanged"
+	NodeTaskTypeNodeVersionChanged    NodeTaskType = "nodeVersionChanged"
+	NodeTaskTypeScriptsChanged        NodeTaskType = "scriptsChanged"
+	NodeTaskTypeNodeLevelChanged      NodeTaskType = "nodeLevelChanged"
 
 	// NS相关
 

internal/db/models/node_value_dao.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"encoding/json"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
@@ -9,6 +10,7 @@ import (
 	"github.com/iwind/TeaGo/maps"
 	"github.com/iwind/TeaGo/types"
 	timeutil "github.com/iwind/TeaGo/utils/time"
+	"strings"
 	"time"
 )
 
@@ -35,9 +37,9 @@ func init() {
 
 // CreateValue 创建值
 func (this *NodeValueDAO) CreateValue(tx *dbs.Tx, clusterId int64, role nodeconfigs.NodeRole, nodeId int64, item string, valueJSON []byte, createdAt int64) error {
-	day := timeutil.FormatTime("Ymd", createdAt)
+	var day = timeutil.FormatTime("Ymd", createdAt)
-	hour := timeutil.FormatTime("YmdH", createdAt)
+	var hour = timeutil.FormatTime("YmdH", createdAt)
-	minute := timeutil.FormatTime("YmdHi", createdAt)
+	var minute = timeutil.FormatTime("YmdHi", createdAt)
 
 	return this.Query(tx).
 		InsertOrUpdateQuickly(maps.Map{
@@ -120,6 +122,17 @@ func (this *NodeValueDAO) ListValuesWithClusterId(tx *dbs.Tx, role string, clust
 
 	_, err = query.Slice(&result).
 		FindAll()
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, nodeValue := range result {
+		nodeValue.Value, _ = json.Marshal(maps.Map{
+			key: types.Float32(string(nodeValue.Value)),
+		})
+	}
+
 	return
 }
 
@@ -144,6 +157,17 @@ func (this *NodeValueDAO) ListValuesForUserNodes(tx *dbs.Tx, item string, key st
 
 	_, err = query.Slice(&result).
 		FindAll()
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, nodeValue := range result {
+		nodeValue.Value, _ = json.Marshal(maps.Map{
+			key: types.Float32(string(nodeValue.Value)),
+		})
+	}
+
 	return
 }
 
@@ -168,9 +192,48 @@ func (this *NodeValueDAO) ListValuesForNSNodes(tx *dbs.Tx, item string, key stri
 
 	_, err = query.Slice(&result).
 		FindAll()
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, nodeValue := range result {
+		nodeValue.Value, _ = json.Marshal(maps.Map{
+			key: types.Float32(string(nodeValue.Value)),
+		})
+	}
+
 	return
 }
 
+// SumAllNodeValues 计算所有节点的某项参数值
+func (this *NodeValueDAO) SumAllNodeValues(tx *dbs.Tx, role string, item nodeconfigs.NodeValueItem, param string, duration int32, durationUnit nodeconfigs.NodeValueDurationUnit) (total float64, avg float64, max float64, err error) {
+	if duration <= 0 {
+		return 0, 0, 0, nil
+	}
+
+	var query = this.Query(tx).
+		Result("SUM(JSON_EXTRACT(value, '$."+param+"')) AS sumValue", "AVG(JSON_EXTRACT(value, '$."+param+"')) AS avgValue", "MAX(JSON_EXTRACT(value, '$."+param+"')) AS maxValueResult"). // maxValue 是个MySQL Keyword，这里使用maxValueResult代替
+		Attr("role", role).
+		Attr("item", item)
+
+	switch durationUnit {
+	case nodeconfigs.NodeValueDurationUnitMinute:
+		fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(duration*60))
+		query.Attr("minute", fromMinute)
+	default:
+		fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(duration*60))
+		query.Attr("minute", fromMinute)
+	}
+
+	m, _, err := query.FindOne()
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	return m.GetFloat64("sumValue"), m.GetFloat64("avgValue"), m.GetFloat64("maxValueResult"), nil
+}
+
 // SumNodeValues 计算节点的某项参数值
 func (this *NodeValueDAO) SumNodeValues(tx *dbs.Tx, role string, nodeId int64, item string, param string, method nodeconfigs.NodeValueSumMethod, duration int32, durationUnit nodeconfigs.NodeValueDurationUnit) (float64, error) {
 	if duration <= 0 {
@@ -261,11 +324,13 @@ func (this *NodeValueDAO) SumNodeClusterValues(tx *dbs.Tx, role string, clusterI
 
 // FindLatestNodeValue 获取最近一条数据
 func (this *NodeValueDAO) FindLatestNodeValue(tx *dbs.Tx, role string, nodeId int64, item string) (*NodeValue, error) {
+	fromMinute := timeutil.FormatTime("YmdHi", time.Now().Unix()-int64(60))
+
 	one, err := this.Query(tx).
 		Attr("role", role).
 		Attr("nodeId", nodeId).
 		Attr("item", item).
-		DescPk().
+		Attr("minute", fromMinute).
 		Find()
 	if err != nil {
 		return nil, err
@@ -275,3 +340,62 @@ func (this *NodeValueDAO) FindLatestNodeValue(tx *dbs.Tx, role string, nodeId in
 	}
 	return one.(*NodeValue), nil
 }
+
+// ComposeNodeStatus 组合节点状态值
+func (this *NodeValueDAO) ComposeNodeStatus(tx *dbs.Tx, role string, nodeId int64, statusConfig *nodeconfigs.NodeStatus) error {
+	var items = []string{
+		nodeconfigs.NodeValueItemCPU,
+		nodeconfigs.NodeValueItemMemory,
+		nodeconfigs.NodeValueItemLoad,
+		nodeconfigs.NodeValueItemTrafficOut,
+		nodeconfigs.NodeValueItemTrafficIn,
+	}
+	ones, err := this.Query(tx).
+		Result("item", "value").
+		Attr("role", role).
+		Attr("nodeId", nodeId).
+		Attr("minute", timeutil.FormatTime("YmdHi", time.Now().Unix()-60)).
+		Where("item IN ('" + strings.Join(items, "', '") + "')").
+		FindAll()
+	if err != nil {
+		return err
+	}
+	for _, one := range ones {
+		var oneValue = one.(*NodeValue)
+		var valueMap = oneValue.DecodeMapValue()
+		switch oneValue.Item {
+		case nodeconfigs.NodeValueItemCPU:
+			statusConfig.CPUUsage = valueMap.GetFloat64("usage")
+		case nodeconfigs.NodeValueItemMemory:
+			statusConfig.MemoryUsage = valueMap.GetFloat64("usage")
+		case nodeconfigs.NodeValueItemLoad:
+			statusConfig.Load1m = valueMap.GetFloat64("load1m")
+			statusConfig.Load5m = valueMap.GetFloat64("load5m")
+			statusConfig.Load15m = valueMap.GetFloat64("load15m")
+		case nodeconfigs.NodeValueItemTrafficOut:
+			statusConfig.TrafficOutBytes = valueMap.GetUint64("total")
+		case nodeconfigs.NodeValueItemTrafficIn:
+			statusConfig.TrafficInBytes = valueMap.GetUint64("total")
+		}
+	}
+
+	return nil
+}
+
+// ComposeNodeStatusJSON 组合节点状态值，并转换为JSON数据
+func (this *NodeValueDAO) ComposeNodeStatusJSON(tx *dbs.Tx, role string, nodeId int64, statusJSON []byte) ([]byte, error) {
+	var statusConfig = &nodeconfigs.NodeStatus{}
+	if len(statusJSON) > 0 {
+		err := json.Unmarshal(statusJSON, statusConfig)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	err := this.ComposeNodeStatus(tx, role, nodeId, statusConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return json.Marshal(statusConfig)
+}

internal/db/models/node_value_dao_test.go

@@ -1,16 +1,21 @@
-package models
+package models_test
 
 import (
+	"github.com/TeaOSLab/EdgeAPI/internal/db/models"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
 	_ "github.com/go-sql-driver/mysql"
 	_ "github.com/iwind/TeaGo/bootstrap"
+	"github.com/iwind/TeaGo/dbs"
+	"github.com/iwind/TeaGo/logs"
 	"github.com/iwind/TeaGo/maps"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
 	"testing"
 	"time"
 )
 
 func TestNodeValueDAO_CreateValue(t *testing.T) {
-	var dao = NewNodeValueDAO()
+	var dao = models.NewNodeValueDAO()
 	m := maps.Map{
 		"hello": "world12344",
 	}
@@ -22,10 +27,55 @@ func TestNodeValueDAO_CreateValue(t *testing.T) {
 }
 
 func TestNodeValueDAO_Clean(t *testing.T) {
-	var dao = NewNodeValueDAO()
+	var dao = models.NewNodeValueDAO()
 	err := dao.Clean(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	t.Log("ok")
 }
+
+func TestNodeValueDAO_CreateManyValues(t *testing.T) {
+	var dao = models.NewNodeValueDAO()
+	var tx *dbs.Tx
+
+	for i := 0; i < 1; i++ {
+		if i%10000 == 0 {
+			t.Log(i)
+		}
+		var item = "connections" + types.String(i)
+		var clusterId int64 = 42
+		var nodeId = rands.Int(1, 100)
+		err := dao.CreateValue(tx, clusterId, nodeconfigs.NodeRoleNode, int64(nodeId), item, []byte(`{"total":1}`), time.Now().Unix())
+		if err != nil {
+			t.Fatal("item: " + item + ", err: " + err.Error())
+		}
+	}
+	t.Log("finished")
+}
+
+func TestNodeValueDAO_SumAllNodeValues(t *testing.T) {
+	var dao = models.NewNodeValueDAO()
+	sum, avg, max, err := dao.SumAllNodeValues(nil, nodeconfigs.NodeRoleNode, nodeconfigs.NodeValueItemCPU, "usage", 1, nodeconfigs.NodeValueDurationUnitMinute)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("sum:", sum, "avg:", avg, "max:", max)
+}
+
+func TestNodeValueDAO_ComposeNodeStatus(t *testing.T) {
+	var dao = models.NewNodeValueDAO()
+	one, err := dao.Query(nil).DescPk().Find()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if one != nil {
+		var config = &nodeconfigs.NodeStatus{}
+		err = dao.ComposeNodeStatus(nil, one.(*models.NodeValue).Role, int64(one.(*models.NodeValue).NodeId), config)
+		if err != nil {
+			t.Fatal(err)
+		}
+		logs.PrintAsJSON(config, t)
+	}
+}

internal/db/models/ns_access_log_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	_ "github.com/go-sql-driver/mysql"
@@ -198,7 +199,7 @@ func (this *NSAccessLogDAO) listAccessLogs(tx *dbs.Tx, lastRequestId string, siz
 			// keyword
 			if len(keyword) > 0 {
 				query.Where("(JSON_EXTRACT(content, '$.remoteAddr') LIKE :keyword OR JSON_EXTRACT(content, '$.questionName') LIKE :keyword OR JSON_EXTRACT(content, '$.recordValue') LIKE :keyword)").
-					Param("keyword", "%"+keyword+"%")
+					Param("keyword", dbutils.QuoteLike(keyword))
 			}
 
 			if !reverse {

internal/db/models/ns_node_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
@@ -51,16 +52,23 @@ func (this *NSNodeDAO) EnableNSNode(tx *dbs.Tx, id int64) error {
 }
 
 // DisableNSNode 禁用条目
-func (this *NSNodeDAO) DisableNSNode(tx *dbs.Tx, id int64) error {
+func (this *NSNodeDAO) DisableNSNode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", NSNodeStateDisabled).
 		Update()
 
 	if err != nil {
 		return err
 	}
-	return this.NotifyUpdate(tx, id)
+
+	err = this.NotifyUpdate(tx, nodeId)
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleDNS, nodeId)
 }
 
 // FindEnabledNSNode 查找启用中的条目
@@ -139,7 +147,7 @@ func (this *NSNodeDAO) CountAllEnabledNodesMatch(tx *dbs.Tx, clusterId int64, in
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 
 	return query.
@@ -176,7 +184,7 @@ func (this *NSNodeDAO) ListAllEnabledNodesMatch(tx *dbs.Tx, clusterId int64, ins
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword)").
-			Param("keyword", "%"+keyword+"%")
+			Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	_, err = query.
 		State(NSNodeStateEnabled).
@@ -338,13 +346,19 @@ func (this *NSNodeDAO) UpdateNodeIsInstalled(tx *dbs.Tx, nodeId int64, isInstall
 }
 
 // UpdateNodeStatus 更改节点状态
-func (this NSNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, statusJSON []byte) error {
+func (this NSNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, nodeStatus *nodeconfigs.NodeStatus) error {
-	if statusJSON == nil {
+	if nodeStatus == nil {
 		return nil
 	}
-	_, err := this.Query(tx).
+
+	nodeStatusJSON, err := json.Marshal(nodeStatus)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.Query(tx).
 		Pk(nodeId).
-		Set("status", string(statusJSON)).
+		Set("status", nodeStatusJSON).
 		Update()
 	return err
 }

internal/db/models/origin_dao.go

@@ -101,7 +101,8 @@ func (this *OriginDAO) CreateOrigin(tx *dbs.Tx,
 	maxIdleConns int32,
 	certRef *sslconfigs.SSLCertRef,
 	domains []string,
-	host string) (originId int64, err error) {
+	host string,
+	followPort bool) (originId int64, err error) {
 	var op = NewOriginOperator()
 	op.AdminId = adminId
 	op.UserId = userId
@@ -167,6 +168,7 @@ func (this *OriginDAO) CreateOrigin(tx *dbs.Tx,
 	}
 
 	op.Host = host
+	op.FollowPort = followPort
 
 	op.State = OriginStateEnabled
 	err = this.Save(tx, op)
@@ -191,7 +193,8 @@ func (this *OriginDAO) UpdateOrigin(tx *dbs.Tx,
 	maxIdleConns int32,
 	certRef *sslconfigs.SSLCertRef,
 	domains []string,
-	host string) error {
+	host string,
+	followPort bool) error {
 	if originId <= 0 {
 		return errors.New("invalid originId")
 	}
@@ -262,6 +265,7 @@ func (this *OriginDAO) UpdateOrigin(tx *dbs.Tx,
 	}
 
 	op.Host = host
+	op.FollowPort = followPort
 
 	err := this.Save(tx, op)
 	if err != nil {
@@ -304,10 +308,11 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 		RequestURI:   origin.HttpRequestURI,
 		RequestHost:  origin.Host,
 		Domains:      origin.DecodeDomains(),
+		FollowPort:   origin.FollowPort,
 	}
 
 	if IsNotNull(origin.Addr) {
-		addr := &serverconfigs.NetworkAddressConfig{}
+		var addr = &serverconfigs.NetworkAddressConfig{}
 		err = json.Unmarshal(origin.Addr, addr)
 		if err != nil {
 			return nil, err
@@ -316,7 +321,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	}
 
 	if IsNotNull(origin.ConnTimeout) {
-		connTimeout := &shared.TimeDuration{}
+		var connTimeout = &shared.TimeDuration{}
 		err = json.Unmarshal(origin.ConnTimeout, &connTimeout)
 		if err != nil {
 			return nil, err
@@ -325,7 +330,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	}
 
 	if IsNotNull(origin.ReadTimeout) {
-		readTimeout := &shared.TimeDuration{}
+		var readTimeout = &shared.TimeDuration{}
 		err = json.Unmarshal(origin.ReadTimeout, &readTimeout)
 		if err != nil {
 			return nil, err
@@ -334,7 +339,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	}
 
 	if IsNotNull(origin.IdleTimeout) {
-		idleTimeout := &shared.TimeDuration{}
+		var idleTimeout = &shared.TimeDuration{}
 		err = json.Unmarshal(origin.IdleTimeout, &idleTimeout)
 		if err != nil {
 			return nil, err
@@ -363,7 +368,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	}
 
 	if IsNotNull(origin.HttpResponseHeader) {
-		ref := &shared.HTTPHeaderPolicyRef{}
+		var ref = &shared.HTTPHeaderPolicyRef{}
 		err = json.Unmarshal(origin.HttpResponseHeader, ref)
 		if err != nil {
 			return nil, err
@@ -382,7 +387,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	}
 
 	if IsNotNull(origin.HealthCheck) {
-		healthCheck := &serverconfigs.HealthCheckConfig{}
+		var healthCheck = &serverconfigs.HealthCheckConfig{}
 		err = json.Unmarshal(origin.HealthCheck, healthCheck)
 		if err != nil {
 			return nil, err
@@ -391,7 +396,7 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	}
 
 	if IsNotNull(origin.Cert) {
-		ref := &sslconfigs.SSLCertRef{}
+		var ref = &sslconfigs.SSLCertRef{}
 		err = json.Unmarshal(origin.Cert, ref)
 		if err != nil {
 			return nil, err
@@ -417,6 +422,19 @@ func (this *OriginDAO) ComposeOriginConfig(tx *dbs.Tx, originId int64, cacheMap
 	return config, nil
 }
 
+// CheckUserOrigin 检查源站权限
+func (this *OriginDAO) CheckUserOrigin(tx *dbs.Tx, userId int64, originId int64) error {
+	reverseProxyId, err := SharedReverseProxyDAO.FindReverseProxyContainsOriginId(tx, originId)
+	if err != nil {
+		return err
+	}
+	if reverseProxyId == 0 {
+		// 这里我们不允许源站没有被使用
+		return ErrNotFound
+	}
+	return SharedReverseProxyDAO.CheckUserReverseProxy(tx, userId, reverseProxyId)
+}
+
 // NotifyUpdate 通知更新
 func (this *OriginDAO) NotifyUpdate(tx *dbs.Tx, originId int64) error {
 	reverseProxyId, err := SharedReverseProxyDAO.FindReverseProxyContainsOriginId(tx, originId)

internal/db/models/origin_model.go

@@ -29,6 +29,7 @@ type Origin struct {
 	Ftp                dbs.JSON `field:"ftp"`                // FTP相关设置
 	CreatedAt          uint64   `field:"createdAt"`          // 创建时间
 	Domains            dbs.JSON `field:"domains"`            // 所属域名
+	FollowPort         bool     `field:"followPort"`         // 端口跟随
 	State              uint8    `field:"state"`              // 状态
 }
 
@@ -58,6 +59,7 @@ type OriginOperator struct {
 	Ftp                interface{} // FTP相关设置
 	CreatedAt          interface{} // 创建时间
 	Domains            interface{} // 所属域名
+	FollowPort         interface{} // 端口跟随
 	State              interface{} // 状态
 }
 

internal/db/models/plan_dao.go

@@ -217,7 +217,7 @@ func (this *PlanDAO) ListEnabledPlans(tx *dbs.Tx, offset int64, size int64) (res
 		Limit(size).
 		Slice(&result).
 		Desc("order").
-		DescPk().
+		AscPk().
 		FindAll()
 	return
 }

internal/db/models/report_node_dao.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"encoding/json"
+	dbutils "github.com/TeaOSLab/EdgeAPI/internal/db/utils"
 	"github.com/TeaOSLab/EdgeAPI/internal/errors"
 	"github.com/TeaOSLab/EdgeAPI/internal/utils"
 	"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
@@ -49,12 +50,17 @@ func (this *ReportNodeDAO) EnableReportNode(tx *dbs.Tx, id int64) error {
 }
 
 // DisableReportNode 禁用条目
-func (this *ReportNodeDAO) DisableReportNode(tx *dbs.Tx, id int64) error {
+func (this *ReportNodeDAO) DisableReportNode(tx *dbs.Tx, nodeId int64) error {
 	_, err := this.Query(tx).
-		Pk(id).
+		Pk(nodeId).
 		Set("state", ReportNodeStateDisabled).
 		Update()
-	return err
+	if err != nil {
+		return err
+	}
+
+	// 删除运行日志
+	return SharedNodeLogDAO.DeleteNodeLogs(tx, nodeconfigs.NodeRoleReport, nodeId)
 }
 
 // FindEnabledReportNode 查找启用中的条目
@@ -169,7 +175,7 @@ func (this *ReportNodeDAO) CountAllEnabledReportNodes(tx *dbs.Tx, groupId int64,
 	}
 	if len(keyword) > 0 {
 		query.Where("(name LIKE :keyword OR location LIKE :keyword OR isp LIKE :keyword OR allowIPs LIKE :keyword OR (status IS NOT NULL AND JSON_EXTRACT(status, 'ip') LIKE :keyword))")
-		query.Param("keyword", "%"+keyword+"%")
+		query.Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	return query.Count()
 }
@@ -201,7 +207,7 @@ func (this *ReportNodeDAO) ListEnabledReportNodes(tx *dbs.Tx, groupId int64, key
 			OR (LENGTH(location)=0 AND JSON_EXTRACT(status, '$.location') LIKE :keyword) 
 			OR (LENGTH(isp)=0 AND JSON_EXTRACT(status, '$.isp') LIKE :keyword)
        ))`)
-		query.Param("keyword", "%"+keyword+"%")
+		query.Param("keyword", dbutils.QuoteLike(keyword))
 	}
 	query.Slice(&result)
 	_, err = query.Asc("isActive").
@@ -263,13 +269,19 @@ func (this *ReportNodeDAO) FindEnabledNodeIdWithUniqueId(tx *dbs.Tx, uniqueId st
 }
 
 // UpdateNodeStatus 更改节点状态
-func (this ReportNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, statusJSON []byte) error {
+func (this ReportNodeDAO) UpdateNodeStatus(tx *dbs.Tx, nodeId int64, nodeStatus *reporterconfigs.Status) error {
-	if statusJSON == nil {
+	if nodeStatus == nil {
 		return nil
 	}
-	_, err := this.Query(tx).
+
+	nodeStatusJSON, err := json.Marshal(nodeStatus)
+	if err != nil {
+		return err
+	}
+
+	_, err = this.Query(tx).
 		Pk(nodeId).
-		Set("status", string(statusJSON)).
+		Set("status", nodeStatusJSON).
 		Update()
 	return err
 }

internal/db/models/reverse_proxy_dao.go

@@ -99,17 +99,18 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 		return nil, nil
 	}
 
-	config := &serverconfigs.ReverseProxyConfig{}
+	var config = &serverconfigs.ReverseProxyConfig{}
 	config.Id = int64(reverseProxy.Id)
 	config.IsOn = reverseProxy.IsOn
 	config.RequestHostType = types.Int8(reverseProxy.RequestHostType)
 	config.RequestHost = reverseProxy.RequestHost
+	config.RequestHostExcludingPort = reverseProxy.RequestHostExcludingPort
 	config.RequestURI = reverseProxy.RequestURI
 	config.StripPrefix = reverseProxy.StripPrefix
 	config.AutoFlush = reverseProxy.AutoFlush == 1
 	config.FollowRedirects = reverseProxy.FollowRedirects == 1
 
-	schedulingConfig := &serverconfigs.SchedulingConfig{}
+	var schedulingConfig = &serverconfigs.SchedulingConfig{}
 	if IsNotNull(reverseProxy.Scheduling) {
 		err = json.Unmarshal(reverseProxy.Scheduling, schedulingConfig)
 		if err != nil {
@@ -118,7 +119,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 		config.Scheduling = schedulingConfig
 	}
 	if IsNotNull(reverseProxy.PrimaryOrigins) {
-		originRefs := []*serverconfigs.OriginRef{}
+		var originRefs = []*serverconfigs.OriginRef{}
 		err = json.Unmarshal(reverseProxy.PrimaryOrigins, &originRefs)
 		if err != nil {
 			return nil, err
@@ -135,13 +136,13 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 	}
 
 	if IsNotNull(reverseProxy.BackupOrigins) {
-		originRefs := []*serverconfigs.OriginRef{}
+		var originRefs = []*serverconfigs.OriginRef{}
 		err = json.Unmarshal(reverseProxy.BackupOrigins, &originRefs)
 		if err != nil {
 			return nil, err
 		}
-		for _, originConfig := range originRefs {
+		for _, ref := range originRefs {
-			originConfig, err := SharedOriginDAO.ComposeOriginConfig(tx, originConfig.OriginId, cacheMap)
+			originConfig, err := SharedOriginDAO.ComposeOriginConfig(tx, ref.OriginId, cacheMap)
 			if err != nil {
 				return nil, err
 			}
@@ -153,7 +154,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 
 	// add headers
 	if IsNotNull(reverseProxy.AddHeaders) {
-		addHeaders := []string{}
+		var addHeaders = []string{}
 		err = json.Unmarshal(reverseProxy.AddHeaders, &addHeaders)
 		if err != nil {
 			return nil, err
@@ -166,7 +167,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 	config.MaxIdleConns = int(reverseProxy.MaxIdleConns)
 
 	if IsNotNull(reverseProxy.ConnTimeout) {
-		connTimeout := &shared.TimeDuration{}
+		var connTimeout = &shared.TimeDuration{}
 		err = json.Unmarshal(reverseProxy.ConnTimeout, &connTimeout)
 		if err != nil {
 			return nil, err
@@ -175,7 +176,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 	}
 
 	if IsNotNull(reverseProxy.ReadTimeout) {
-		readTimeout := &shared.TimeDuration{}
+		var readTimeout = &shared.TimeDuration{}
 		err = json.Unmarshal(reverseProxy.ReadTimeout, &readTimeout)
 		if err != nil {
 			return nil, err
@@ -184,7 +185,7 @@ func (this *ReverseProxyDAO) ComposeReverseProxyConfig(tx *dbs.Tx, reverseProxyI
 	}
 
 	if IsNotNull(reverseProxy.IdleTimeout) {
-		idleTimeout := &shared.TimeDuration{}
+		var idleTimeout = &shared.TimeDuration{}
 		err = json.Unmarshal(reverseProxy.IdleTimeout, &idleTimeout)
 		if err != nil {
 			return nil, err
@@ -304,6 +305,7 @@ func (this *ReverseProxyDAO) UpdateReverseProxy(tx *dbs.Tx,
 	reverseProxyId int64,
 	requestHostType int8,
 	requestHost string,
+	requestHostExcludingPort bool,
 	requestURI string,
 	stripPrefix string,
 	autoFlush bool,
@@ -328,6 +330,7 @@ func (this *ReverseProxyDAO) UpdateReverseProxy(tx *dbs.Tx,
 	op.RequestHostType = requestHostType
 
 	op.RequestHost = requestHost
+	op.RequestHostExcludingPort = requestHostExcludingPort
 	op.RequestURI = requestURI
 	op.StripPrefix = stripPrefix
 	op.AutoFlush = autoFlush

internal/db/models/reverse_proxy_model.go

@@ -4,55 +4,57 @@ import "github.com/iwind/TeaGo/dbs"
 
 // ReverseProxy 反向代理配置
 type ReverseProxy struct {
-	Id              uint32   `field:"id"`              // ID
+	Id                       uint32   `field:"id"`                       // ID
-	AdminId         uint32   `field:"adminId"`         // 管理员ID
+	AdminId                  uint32   `field:"adminId"`                  // 管理员ID
-	UserId          uint32   `field:"userId"`          // 用户ID
+	UserId                   uint32   `field:"userId"`                   // 用户ID
-	TemplateId      uint32   `field:"templateId"`      // 模版ID
+	TemplateId               uint32   `field:"templateId"`               // 模版ID
-	IsOn            bool     `field:"isOn"`            // 是否启用
+	IsOn                     bool     `field:"isOn"`                     // 是否启用
-	Scheduling      dbs.JSON `field:"scheduling"`      // 调度算法
+	Scheduling               dbs.JSON `field:"scheduling"`               // 调度算法
-	PrimaryOrigins  dbs.JSON `field:"primaryOrigins"`  // 主要源站
+	PrimaryOrigins           dbs.JSON `field:"primaryOrigins"`           // 主要源站
-	BackupOrigins   dbs.JSON `field:"backupOrigins"`   // 备用源站
+	BackupOrigins            dbs.JSON `field:"backupOrigins"`            // 备用源站
-	StripPrefix     string   `field:"stripPrefix"`     // 去除URL前缀
+	StripPrefix              string   `field:"stripPrefix"`              // 去除URL前缀
-	RequestHostType uint8    `field:"requestHostType"` // 请求Host类型
+	RequestHostType          uint8    `field:"requestHostType"`          // 请求Host类型
-	RequestHost     string   `field:"requestHost"`     // 请求Host
+	RequestHost              string   `field:"requestHost"`              // 请求Host
-	RequestURI      string   `field:"requestURI"`      // 请求URI
+	RequestHostExcludingPort bool     `field:"requestHostExcludingPort"` // 移除请求Host中的域名
-	AutoFlush       uint8    `field:"autoFlush"`       // 是否自动刷新缓冲区
+	RequestURI               string   `field:"requestURI"`               // 请求URI
-	AddHeaders      dbs.JSON `field:"addHeaders"`      // 自动添加的Header列表
+	AutoFlush                uint8    `field:"autoFlush"`                // 是否自动刷新缓冲区
-	State           uint8    `field:"state"`           // 状态
+	AddHeaders               dbs.JSON `field:"addHeaders"`               // 自动添加的Header列表
-	CreatedAt       uint64   `field:"createdAt"`       // 创建时间
+	State                    uint8    `field:"state"`                    // 状态
-	ConnTimeout     dbs.JSON `field:"connTimeout"`     // 连接超时时间
+	CreatedAt                uint64   `field:"createdAt"`                // 创建时间
-	ReadTimeout     dbs.JSON `field:"readTimeout"`     // 读取超时时间
+	ConnTimeout              dbs.JSON `field:"connTimeout"`              // 连接超时时间
-	IdleTimeout     dbs.JSON `field:"idleTimeout"`     // 空闲超时时间
+	ReadTimeout              dbs.JSON `field:"readTimeout"`              // 读取超时时间
-	MaxConns        uint32   `field:"maxConns"`        // 最大并发连接数
+	IdleTimeout              dbs.JSON `field:"idleTimeout"`              // 空闲超时时间
-	MaxIdleConns    uint32   `field:"maxIdleConns"`    // 最大空闲连接数
+	MaxConns                 uint32   `field:"maxConns"`                 // 最大并发连接数
-	ProxyProtocol   dbs.JSON `field:"proxyProtocol"`   // Proxy Protocol配置
+	MaxIdleConns             uint32   `field:"maxIdleConns"`             // 最大空闲连接数
-	FollowRedirects uint8    `field:"followRedirects"` // 回源跟随
+	ProxyProtocol            dbs.JSON `field:"proxyProtocol"`            // Proxy Protocol配置
+	FollowRedirects          uint8    `field:"followRedirects"`          // 回源跟随
 }
 
 type ReverseProxyOperator struct {
-	Id              interface{} // ID
+	Id                       interface{} // ID
-	AdminId         interface{} // 管理员ID
+	AdminId                  interface{} // 管理员ID
-	UserId          interface{} // 用户ID
+	UserId                   interface{} // 用户ID
-	TemplateId      interface{} // 模版ID
+	TemplateId               interface{} // 模版ID
-	IsOn            interface{} // 是否启用
+	IsOn                     interface{} // 是否启用
-	Scheduling      interface{} // 调度算法
+	Scheduling               interface{} // 调度算法
-	PrimaryOrigins  interface{} // 主要源站
+	PrimaryOrigins           interface{} // 主要源站
-	BackupOrigins   interface{} // 备用源站
+	BackupOrigins            interface{} // 备用源站
-	StripPrefix     interface{} // 去除URL前缀
+	StripPrefix              interface{} // 去除URL前缀
-	RequestHostType interface{} // 请求Host类型
+	RequestHostType          interface{} // 请求Host类型
-	RequestHost     interface{} // 请求Host
+	RequestHost              interface{} // 请求Host
-	RequestURI      interface{} // 请求URI
+	RequestHostExcludingPort interface{} // 移除请求Host中的域名
-	AutoFlush       interface{} // 是否自动刷新缓冲区
+	RequestURI               interface{} // 请求URI
-	AddHeaders      interface{} // 自动添加的Header列表
+	AutoFlush                interface{} // 是否自动刷新缓冲区
-	State           interface{} // 状态
+	AddHeaders               interface{} // 自动添加的Header列表
-	CreatedAt       interface{} // 创建时间
+	State                    interface{} // 状态
-	ConnTimeout     interface{} // 连接超时时间
+	CreatedAt                interface{} // 创建时间
-	ReadTimeout     interface{} // 读取超时时间
+	ConnTimeout              interface{} // 连接超时时间
-	IdleTimeout     interface{} // 空闲超时时间
+	ReadTimeout              interface{} // 读取超时时间
-	MaxConns        interface{} // 最大并发连接数
+	IdleTimeout              interface{} // 空闲超时时间
-	MaxIdleConns    interface{} // 最大空闲连接数
+	MaxConns                 interface{} // 最大并发连接数
-	ProxyProtocol   interface{} // Proxy Protocol配置
+	MaxIdleConns             interface{} // 最大空闲连接数
-	FollowRedirects interface{} // 回源跟随
+	ProxyProtocol            interface{} // Proxy Protocol配置
+	FollowRedirects          interface{} // 回源跟随
 }
 
 func NewReverseProxyOperator() *ReverseProxyOperator {

internal/db/models/script_dao_plus_test.go

@@ -0,0 +1,6 @@
+package models_test
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+)

internal/db/models/script_history_dao_plus_test.go

@@ -0,0 +1,6 @@
+package models_test
+
+import (
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/iwind/TeaGo/bootstrap"
+)

internal/db/models/script_history_model.go

@@ -0,0 +1,24 @@
+package models
+
+// ScriptHistory 脚本历史记录
+type ScriptHistory struct {
+	Id       uint32 `field:"id"`       // ID
+	UserId   uint64 `field:"userId"`   // 用户ID
+	ScriptId uint64 `field:"scriptId"` // 脚本ID
+	Filename string `field:"filename"` // 文件名
+	Code     string `field:"code"`     // 代码
+	Version  uint64 `field:"version"`  // 版本号
+}
+
+type ScriptHistoryOperator struct {
+	Id       interface{} // ID
+	UserId   interface{} // 用户ID
+	ScriptId interface{} // 脚本ID
+	Filename interface{} // 文件名
+	Code     interface{} // 代码
+	Version  interface{} // 版本号
+}
+
+func NewScriptHistoryOperator() *ScriptHistoryOperator {
+	return &ScriptHistoryOperator{}
+}

internal/db/models/script_history_model_ext.go

@@ -0,0 +1 @@
+package models

internal/db/models/script_model.go

@@ -0,0 +1,30 @@
+package models
+
+// Script 脚本库
+type Script struct {
+	Id        uint64 `field:"id"`        // ID
+	UserId    uint64 `field:"userId"`    // 用户ID
+	IsOn      bool   `field:"isOn"`      // 是否启用
+	Name      string `field:"name"`      // 名称
+	Filename  string `field:"filename"`  // 文件名
+	Code      string `field:"code"`      // 代码
+	CreatedAt uint64 `field:"createdAt"` // 创建时间
+	UpdatedAt uint64 `field:"updatedAt"` // 修改时间
+	State     uint8  `field:"state"`     // 是否启用
+}
+
+type ScriptOperator struct {
+	Id        interface{} // ID
+	UserId    interface{} // 用户ID
+	IsOn      interface{} // 是否启用
+	Name      interface{} // 名称
+	Filename  interface{} // 文件名
+	Code      interface{} // 代码
+	CreatedAt interface{} // 创建时间
+	UpdatedAt interface{} // 修改时间
+	State     interface{} // 是否启用
+}
+
+func NewScriptOperator() *ScriptOperator {
+	return &ScriptOperator{}
+}