Compare commits
38 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b0d1f2c8ea | ||
|
|
94de773773 | ||
|
|
da8d7f1437 | ||
|
|
777bbdf50b | ||
|
|
f5596b4f0c | ||
|
|
480d165538 | ||
|
|
870d18a5ac | ||
|
|
731f213310 | ||
|
|
f1b3a7463d | ||
|
|
5020985b96 | ||
|
|
a0ac33af5a | ||
|
|
ffe3f248a9 | ||
|
|
02dc42d8e0 | ||
|
|
b68d92d506 | ||
|
|
1144ca8a8b | ||
|
|
0b216090f1 | ||
|
|
b1e609b2b4 | ||
|
|
8de74b62e7 | ||
|
|
02da06dffd | ||
|
|
bd417a3795 | ||
|
|
42e1e0c3df | ||
|
|
216dacc348 | ||
|
|
968eac6046 | ||
|
|
9a3740dc00 | ||
|
|
ef1626de53 | ||
|
|
5eb122f0a3 | ||
|
|
6e2c5aabc2 | ||
|
|
1a1ae239b3 | ||
|
|
95f42ebd37 | ||
|
|
7111d23f11 | ||
|
|
e42c8452ff | ||
|
|
fe6e5ba5f9 | ||
|
|
406d5de482 | ||
|
|
f66e672bb4 | ||
|
|
7252d00c8e | ||
|
|
7ec4656aea | ||
|
|
33156fed2a | ||
|
|
e8538d4e34 |
@@ -9,6 +9,7 @@ import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/nodes"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
_ "github.com/iwind/TeaGo/bootstrap"
|
||||
@@ -22,8 +23,10 @@ import (
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -31,7 +34,7 @@ func main() {
|
||||
var app = apps.NewAppCmd().
|
||||
Version(teaconst.Version).
|
||||
Product(teaconst.ProductName).
|
||||
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|config|pprof|accesslog|uninstall]").
|
||||
Usage(teaconst.ProcessName + " [-v|start|stop|restart|status|quit|test|reload|service|daemon|config|pprof|top|accesslog|uninstall]").
|
||||
Usage(teaconst.ProcessName + " [trackers|goman|conns|gc|bandwidth|disk|cache.garbage]").
|
||||
Usage(teaconst.ProcessName + " [ip.drop|ip.reject|ip.remove|ip.close] IP")
|
||||
|
||||
@@ -552,7 +555,7 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
err = os.WriteFile(Tea.Root + "/configs/api_node.yaml", configYAML, 0666)
|
||||
err = os.WriteFile(Tea.Root+"/configs/api_node.yaml", configYAML, 0666)
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]write config failed: " + err.Error())
|
||||
return
|
||||
@@ -560,6 +563,37 @@ func main() {
|
||||
|
||||
fmt.Println("success")
|
||||
})
|
||||
app.On("top", func() {
|
||||
var sock = gosock.NewTmpSock(teaconst.ProcessName)
|
||||
reply, err := sock.Send(&gosock.Command{Code: "pid"})
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]not started yet")
|
||||
return
|
||||
}
|
||||
|
||||
var pid = maps.NewMap(reply.Params).GetInt("pid")
|
||||
if pid <= 0 {
|
||||
fmt.Println("[ERROR]invalid pid '" + types.String(pid) + "'")
|
||||
return
|
||||
}
|
||||
|
||||
topExe, _ := executils.LookPath("top")
|
||||
if len(topExe) > 0 {
|
||||
if runtime.GOOS == "linux" {
|
||||
err = syscall.Exec(topExe, []string{topExe, "-p", types.String(pid)}, os.Environ())
|
||||
} else if runtime.GOOS == "darwin" {
|
||||
err = syscall.Exec(topExe, []string{topExe, "-pid", types.String(pid)}, os.Environ())
|
||||
} else {
|
||||
fmt.Println("[ERROR]not supported os '" + runtime.GOOS + "'")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println("[ERROR]start failed: " + err.Error())
|
||||
}
|
||||
} else {
|
||||
fmt.Println("[ERROR]could not found 'top' command in this system")
|
||||
}
|
||||
})
|
||||
app.Run(func() {
|
||||
var node = nodes.NewNode()
|
||||
node.Start()
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/files"
|
||||
timeutil "github.com/iwind/TeaGo/utils/time"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2023 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package apps
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"runtime"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"math/big"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
|
||||
@@ -4,9 +4,12 @@ package caches
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -20,19 +23,31 @@ type KVFileList struct {
|
||||
|
||||
onAdd func(item *Item)
|
||||
onRemove func(item *Item)
|
||||
|
||||
memCache *ttlcache.Cache[int64]
|
||||
}
|
||||
|
||||
func NewKVFileList(dir string) *KVFileList {
|
||||
var memGB = memutils.SystemMemoryGB()
|
||||
if memGB <= 0 {
|
||||
memGB = 1
|
||||
}
|
||||
var maxCachePieces = 32
|
||||
var maxCacheItems = memGB << 15
|
||||
|
||||
var memCache = ttlcache.NewCache[int64](ttlcache.NewPiecesOption(maxCachePieces), ttlcache.NewMaxItemsOption(maxCacheItems))
|
||||
|
||||
dir = strings.TrimSuffix(dir, "/")
|
||||
|
||||
var stores = [countKVStores]*KVListFileStore{}
|
||||
for i := 0; i < countKVStores; i++ {
|
||||
stores[i] = NewKVListFileStore(dir + "/db-" + types.String(i) + ".store")
|
||||
stores[i] = NewKVListFileStore(dir+"/db-"+types.String(i)+".store", memCache)
|
||||
}
|
||||
|
||||
return &KVFileList{
|
||||
dir: dir,
|
||||
stores: stores,
|
||||
dir: dir,
|
||||
stores: stores,
|
||||
memCache: memCache,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +74,7 @@ func (this *KVFileList) Init() error {
|
||||
|
||||
// Reset 重置数据
|
||||
func (this *KVFileList) Reset() error {
|
||||
// do nothing
|
||||
this.memCache.Clean()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -74,16 +89,31 @@ func (this *KVFileList) Add(hash string, item *Item) error {
|
||||
this.onAdd(item)
|
||||
}
|
||||
|
||||
if item.ExpiresAt > 0 {
|
||||
this.memCache.Write(hash, item.HeaderSize+item.BodySize, min(item.ExpiresAt, fasttime.Now().Unix()+3600))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exist 检查内容是否存在
|
||||
func (this *KVFileList) Exist(hash string) (bool, int64, error) {
|
||||
// read from cache
|
||||
var cacheItem = this.memCache.Read(hash)
|
||||
if cacheItem != nil {
|
||||
return true, cacheItem.Value, nil
|
||||
}
|
||||
|
||||
return this.getStore(hash).ExistItem(hash)
|
||||
}
|
||||
|
||||
// ExistQuick 快速检查内容是否存在
|
||||
func (this *KVFileList) ExistQuick(hash string) (bool, error) {
|
||||
// read from cache
|
||||
if this.memCache.Read(hash) != nil {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return this.getStore(hash).ExistQuickItem(hash)
|
||||
}
|
||||
|
||||
@@ -150,6 +180,9 @@ func (this *KVFileList) Remove(hash string) error {
|
||||
this.onRemove(nil)
|
||||
}
|
||||
|
||||
// remove from cache
|
||||
this.memCache.Delete(hash)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -204,6 +237,9 @@ func (this *KVFileList) CleanAll() error {
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
|
||||
this.memCache.Clean()
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
@@ -287,6 +323,9 @@ func (this *KVFileList) Close() error {
|
||||
})
|
||||
}
|
||||
group.Wait()
|
||||
|
||||
this.memCache.Destroy()
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/kvstore"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -20,11 +21,14 @@ type KVListFileStore struct {
|
||||
itemsTable *kvstore.Table[*Item]
|
||||
|
||||
rawIsReady bool
|
||||
|
||||
memCache *ttlcache.Cache[int64]
|
||||
}
|
||||
|
||||
func NewKVListFileStore(path string) *KVListFileStore {
|
||||
func NewKVListFileStore(path string, memCache *ttlcache.Cache[int64]) *KVListFileStore {
|
||||
return &KVListFileStore{
|
||||
path: path,
|
||||
path: path,
|
||||
memCache: memCache,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,7 +110,14 @@ func (this *KVListFileStore) ExistItem(hash string) (bool, int64, error) {
|
||||
return false, -1, nil
|
||||
}
|
||||
|
||||
return item.ExpiresAt > fasttime.Now().Unix(), item.HeaderSize + item.BodySize, nil
|
||||
if item.ExpiresAt <= fasttime.Now().Unix() {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
// write to cache
|
||||
this.memCache.Write(hash, item.HeaderSize+item.BodySize, min(item.ExpiresAt, fasttime.Now().Unix()+3600))
|
||||
|
||||
return true, item.HeaderSize + item.BodySize, nil
|
||||
}
|
||||
|
||||
func (this *KVListFileStore) ExistQuickItem(hash string) (bool, error) {
|
||||
@@ -168,6 +179,7 @@ func (this *KVListFileStore) PurgeItems(count int, callback func(hash string) er
|
||||
if deleteErr != nil {
|
||||
return deleteErr
|
||||
}
|
||||
this.memCache.Delete(hash)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -214,6 +226,7 @@ func (this *KVListFileStore) PurgeLFUItems(count int, callback func(hash string)
|
||||
if deleteErr != nil {
|
||||
return deleteErr
|
||||
}
|
||||
this.memCache.Delete(hash)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -276,6 +289,9 @@ func (this *KVListFileStore) CleanItemsWithPrefix(prefix string) error {
|
||||
return false, setErr
|
||||
}
|
||||
|
||||
// remove from cache
|
||||
this.memCache.Delete(item.Key)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -334,6 +350,9 @@ func (this *KVListFileStore) CleanItemsWithWildcardPrefix(prefix string) error {
|
||||
return false, setErr
|
||||
}
|
||||
|
||||
// remove from cache
|
||||
this.memCache.Delete(item.Key)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -402,6 +421,9 @@ func (this *KVListFileStore) CleanItemsWithWildcardKey(key string) error {
|
||||
return false, setErr
|
||||
}
|
||||
|
||||
// remove from cache
|
||||
this.memCache.Delete(item.Key)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/dbs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/ttlcache"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
@@ -38,6 +38,9 @@ func NewOpenFileCache(maxCount int) (*OpenFileCache, error) {
|
||||
if maxCount <= 0 {
|
||||
maxCount = 16384
|
||||
}
|
||||
if maxCount > 65535 {
|
||||
maxCount = 65535
|
||||
}
|
||||
|
||||
var cache = &OpenFileCache{
|
||||
maxCount: maxCount,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// PartialRanges 内容分区范围定义
|
||||
type PartialRanges struct {
|
||||
Version int `json:"version"` // 版本号
|
||||
Ranges [][2]int64 `json:"ranges"` // 范围
|
||||
BodySize int64 `json:"bodySize"` // 总长度
|
||||
Version int `json:"version"` // 版本号
|
||||
Ranges [][2]int64 `json:"ranges"` // 范围
|
||||
BodySize int64 `json:"bodySize"` // 总长度
|
||||
ContentMD5 string `json:"contentMD5"` // 内容md5
|
||||
}
|
||||
|
||||
// NewPartialRanges 获取新对象
|
||||
func NewPartialRanges(expiresAt int64) *PartialRanges {
|
||||
return &PartialRanges{
|
||||
Ranges: [][2]int64{},
|
||||
Version: 1,
|
||||
Version: 2,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +47,8 @@ func NewPartialRangesFromData(data []byte) (*PartialRanges, error) {
|
||||
if commaIndex > 0 {
|
||||
rs.Ranges = append(rs.Ranges, [2]int64{types.Int64(line[colonIndex+1 : commaIndex]), types.Int64(line[commaIndex+1:])})
|
||||
}
|
||||
case "m": // Content-MD5
|
||||
rs.ContentMD5 = string(line[colonIndex+1:])
|
||||
}
|
||||
}
|
||||
data = data[index+1:]
|
||||
@@ -101,7 +105,6 @@ func (this *PartialRanges) Add(begin int64, end int64) {
|
||||
}
|
||||
|
||||
// insert
|
||||
// TODO 将来使用二分法改进
|
||||
var index = -1
|
||||
for i, r := range this.Ranges {
|
||||
if r[0] > begin || (r[0] == begin && r[1] >= end) {
|
||||
@@ -127,7 +130,6 @@ func (this *PartialRanges) Contains(begin int64, end int64) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// TODO 使用二分法查找改进性能
|
||||
for _, r2 := range this.Ranges {
|
||||
if r2[0] <= begin && r2[1] >= end {
|
||||
return true
|
||||
@@ -143,7 +145,6 @@ func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool)
|
||||
return
|
||||
}
|
||||
|
||||
// TODO 使用二分法查找改进性能
|
||||
for _, r2 := range this.Ranges {
|
||||
if r2[0] <= begin && r2[1] > begin {
|
||||
r = [2]int64{begin, this.min(end, r2[1])}
|
||||
@@ -154,10 +155,28 @@ func (this *PartialRanges) Nearest(begin int64, end int64) (r [2]int64, ok bool)
|
||||
return
|
||||
}
|
||||
|
||||
// FindRangeAtPosition 查找在某个位置上的范围
|
||||
func (this *PartialRanges) FindRangeAtPosition(position int64) (r rangeutils.Range, ok bool) {
|
||||
if len(this.Ranges) == 0 || position < 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, r2 := range this.Ranges {
|
||||
if r2[0] <= position && r2[1] > position {
|
||||
return [2]int64{position, r2[1]}, true
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// 转换为字符串
|
||||
func (this *PartialRanges) String() string {
|
||||
var s = "v:" + strconv.Itoa(this.Version) + "\n" + // version
|
||||
"b:" + this.formatInt64(this.BodySize) + "\n" // bodySize
|
||||
if len(this.ContentMD5) > 0 {
|
||||
s += "m:" + this.ContentMD5 + "\n" // Content-MD5
|
||||
}
|
||||
for _, r := range this.Ranges {
|
||||
s += "r:" + this.formatInt64(r[0]) + "," + this.formatInt64(r[1]) + "\n" // range
|
||||
}
|
||||
|
||||
@@ -4,10 +4,10 @@ package caches
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fnv"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/caches"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
@@ -167,8 +169,13 @@ func TestPartialRanges_Encode_String(t *testing.T) {
|
||||
for i := 0; i < 10; i++ {
|
||||
r.Ranges = append(r.Ranges, [2]int64{int64(i * 100), int64(i*100 + 100)})
|
||||
}
|
||||
|
||||
var sum = md5.Sum([]byte("123456"))
|
||||
r.ContentMD5 = base64.StdEncoding.EncodeToString(sum[:])
|
||||
|
||||
var before = time.Now()
|
||||
var data = r.String()
|
||||
t.Log(data)
|
||||
t.Log(time.Since(before).Seconds()*1000, "ms")
|
||||
t.Log(len(data))
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package caches
|
||||
|
||||
import "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
"io"
|
||||
)
|
||||
|
||||
type ReaderFunc func(n int) (goNext bool, err error)
|
||||
|
||||
@@ -41,6 +44,9 @@ type Reader interface {
|
||||
// ContainsRange 是否包含某个区间内容
|
||||
ContainsRange(r rangeutils.Range) (r2 rangeutils.Range, ok bool)
|
||||
|
||||
// SetNextReader 设置下一个内容Reader
|
||||
SetNextReader(nextReader io.ReadCloser)
|
||||
|
||||
// Close 关闭
|
||||
Close() error
|
||||
}
|
||||
|
||||
14
internal/caches/reader_base.go
Normal file
14
internal/caches/reader_base.go
Normal file
@@ -0,0 +1,14 @@
|
||||
// Copyright 2024 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
import "io"
|
||||
|
||||
type BaseReader struct {
|
||||
nextReader io.ReadCloser
|
||||
}
|
||||
|
||||
// SetNextReader 设置下一个内容Reader
|
||||
func (this *BaseReader) SetNextReader(nextReader io.ReadCloser) {
|
||||
this.nextReader = nextReader
|
||||
}
|
||||
@@ -11,7 +11,9 @@ import (
|
||||
)
|
||||
|
||||
type FileReader struct {
|
||||
fp *os.File
|
||||
BaseReader
|
||||
|
||||
fp *fsutils.File
|
||||
|
||||
openFile *OpenFile
|
||||
openFileCache *OpenFileCache
|
||||
@@ -29,7 +31,7 @@ type FileReader struct {
|
||||
isClosed bool
|
||||
}
|
||||
|
||||
func NewFileReader(fp *os.File) *FileReader {
|
||||
func NewFileReader(fp *fsutils.File) *FileReader {
|
||||
return &FileReader{fp: fp}
|
||||
}
|
||||
|
||||
@@ -175,9 +177,7 @@ func (this *FileReader) ReadHeader(buf []byte, callback ReaderFunc) error {
|
||||
var headerSize = this.headerSize
|
||||
|
||||
for {
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
n, err := this.fp.Read(buf)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if n > 0 {
|
||||
if n < headerSize {
|
||||
goNext, e := callback(n)
|
||||
@@ -239,9 +239,7 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
|
||||
}
|
||||
|
||||
for {
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
n, err := this.fp.Read(buf)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if n > 0 {
|
||||
goNext, e := callback(n)
|
||||
if e != nil {
|
||||
@@ -272,9 +270,7 @@ func (this *FileReader) Read(buf []byte) (n int, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
n, err = this.fp.Read(buf)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if err != nil && err != io.EOF {
|
||||
_ = this.discard()
|
||||
}
|
||||
@@ -306,17 +302,14 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
|
||||
isOk = true
|
||||
return ErrInvalidRange
|
||||
}
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
_, err := this.fp.Seek(offset, io.SeekStart)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
n, err := this.fp.Read(buf)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
var n int
|
||||
n, err = this.fp.Read(buf)
|
||||
if n > 0 {
|
||||
var n2 = int(end-offset) + 1
|
||||
if n2 <= n {
|
||||
@@ -352,6 +345,33 @@ func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callba
|
||||
|
||||
isOk = true
|
||||
|
||||
// 读取下一个Reader
|
||||
if this.nextReader != nil {
|
||||
defer func() {
|
||||
_ = this.nextReader.Close()
|
||||
}()
|
||||
|
||||
for {
|
||||
var n int
|
||||
n, err = this.nextReader.Read(buf)
|
||||
if n > 0 {
|
||||
goNext, writeErr := callback(n)
|
||||
if writeErr != nil {
|
||||
return writeErr
|
||||
}
|
||||
if !goNext {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
return err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -362,7 +382,7 @@ func (this *FileReader) ContainsRange(r rangeutils.Range) (r2 rangeutils.Range,
|
||||
|
||||
// FP 原始的文件句柄
|
||||
func (this *FileReader) FP() *os.File {
|
||||
return this.fp
|
||||
return this.fp.Raw()
|
||||
}
|
||||
|
||||
func (this *FileReader) Close() error {
|
||||
@@ -377,7 +397,7 @@ func (this *FileReader) Close() error {
|
||||
} else {
|
||||
var cacheMeta = make([]byte, len(this.meta))
|
||||
copy(cacheMeta, this.meta)
|
||||
this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp, cacheMeta, this.header, this.LastModified(), this.bodySize))
|
||||
this.openFileCache.Put(this.fp.Name(), NewOpenFile(this.fp.Raw(), cacheMeta, this.header, this.LastModified(), this.bodySize))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -385,10 +405,8 @@ func (this *FileReader) Close() error {
|
||||
return this.fp.Close()
|
||||
}
|
||||
|
||||
func (this *FileReader) readToBuff(fp *os.File, buf []byte) (ok bool, err error) {
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
func (this *FileReader) readToBuff(fp *fsutils.File, buf []byte) (ok bool, err error) {
|
||||
n, err := fp.Read(buf)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package caches
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -36,7 +37,7 @@ func TestFileReader(t *testing.T) {
|
||||
defer func() {
|
||||
_ = fp.Close()
|
||||
}()
|
||||
reader := NewFileReader(fp)
|
||||
reader := NewFileReader(fsutils.NewFile(fp, fsutils.FlagRead))
|
||||
err = reader.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -75,7 +76,7 @@ func TestFileReader_ReadHeader(t *testing.T) {
|
||||
defer func() {
|
||||
_ = fp.Close()
|
||||
}()
|
||||
var reader = NewFileReader(fp)
|
||||
var reader = NewFileReader(fsutils.NewFile(fp, fsutils.FlagRead))
|
||||
err = reader.Init()
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -138,7 +139,7 @@ func TestFileReader_Range(t *testing.T) {
|
||||
defer func() {
|
||||
_ = fp.Close()
|
||||
}()
|
||||
reader := NewFileReader(fp)
|
||||
reader := NewFileReader(fsutils.NewFile(fp, fsutils.FlagRead))
|
||||
err = reader.Init()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
)
|
||||
|
||||
type MemoryReader struct {
|
||||
BaseReader
|
||||
|
||||
item *MemoryItem
|
||||
|
||||
offset int
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
rangeutils "github.com/TeaOSLab/EdgeNode/internal/utils/ranges"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type PartialFileReader struct {
|
||||
@@ -18,7 +17,7 @@ type PartialFileReader struct {
|
||||
rangePath string
|
||||
}
|
||||
|
||||
func NewPartialFileReader(fp *os.File) *PartialFileReader {
|
||||
func NewPartialFileReader(fp *fsutils.File) *PartialFileReader {
|
||||
return &PartialFileReader{
|
||||
FileReader: NewFileReader(fp),
|
||||
rangePath: PartialRangesFilePath(fp.Name()),
|
||||
|
||||
@@ -10,15 +10,16 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/bytepool"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
@@ -380,6 +381,7 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
|
||||
|
||||
// 检查文件记录是否已过期
|
||||
var estimatedSize int64
|
||||
var existInList bool
|
||||
if !useStale {
|
||||
exists, filesize, err := this.list.Exist(hash)
|
||||
if err != nil {
|
||||
@@ -389,15 +391,12 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
estimatedSize = filesize
|
||||
existInList = true
|
||||
}
|
||||
|
||||
// 尝试通过MMAP读取
|
||||
if estimatedSize > 0 {
|
||||
if !fsutils.ReaderLimiter.TryAck() {
|
||||
return nil, ErrServerIsBusy
|
||||
}
|
||||
reader, err := this.tryMMAPReader(isPartial, estimatedSize, path)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -416,11 +415,13 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
|
||||
|
||||
var err error
|
||||
if openFile == nil {
|
||||
if !fsutils.ReaderLimiter.TryAck() {
|
||||
return nil, ErrServerIsBusy
|
||||
if existInList {
|
||||
fsutils.ReaderLimiter.Ack()
|
||||
}
|
||||
fp, err = os.OpenFile(path, os.O_RDONLY, 0444)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
if existInList {
|
||||
fsutils.ReaderLimiter.Release()
|
||||
}
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return nil, err
|
||||
@@ -439,12 +440,12 @@ func (this *FileStorage) openReader(key string, allowMemory bool, useStale bool,
|
||||
|
||||
var reader Reader
|
||||
if isPartial {
|
||||
var partialFileReader = NewPartialFileReader(fp)
|
||||
var partialFileReader = NewPartialFileReader(fsutils.NewFile(fp, fsutils.FlagRead))
|
||||
partialFileReader.openFile = openFile
|
||||
partialFileReader.openFileCache = openFileCache
|
||||
reader = partialFileReader
|
||||
} else {
|
||||
var fileReader = NewFileReader(fp)
|
||||
var fileReader = NewFileReader(fsutils.NewFile(fp, fsutils.FlagRead))
|
||||
fileReader.openFile = openFile
|
||||
fileReader.openFileCache = openFileCache
|
||||
reader = fileReader
|
||||
@@ -591,13 +592,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
// 数据库中是否存在
|
||||
existsCacheItem, _, _ := this.list.Exist(hash)
|
||||
if existsCacheItem {
|
||||
if !fsutils.ReaderLimiter.TryAck() {
|
||||
return nil, ErrServerIsBusy
|
||||
}
|
||||
readerFp, err := os.OpenFile(tmpPath, os.O_RDONLY, 0444)
|
||||
fsutils.ReaderLimiter.Release()
|
||||
readerFp, err := fsutils.OpenFile(tmpPath, os.O_RDONLY, 0444)
|
||||
if err == nil {
|
||||
var partialReader = NewPartialFileReader(readerFp)
|
||||
var partialReader = NewPartialFileReader(fsutils.NewFile(readerFp, fsutils.FlagRead))
|
||||
err = partialReader.Init()
|
||||
_ = partialReader.Close()
|
||||
if err == nil && partialReader.bodyOffset > 0 {
|
||||
@@ -628,18 +625,22 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
if isNewCreated && existsFile {
|
||||
flags |= os.O_TRUNC
|
||||
}
|
||||
if !fsutils.WriterLimiter.TryAck() {
|
||||
return nil, ErrServerIsBusy
|
||||
if !isFlushing {
|
||||
if !fsutils.WriterLimiter.TryAck() {
|
||||
return nil, ErrServerIsBusy
|
||||
}
|
||||
}
|
||||
fp, err := os.OpenFile(tmpPath, flags, 0666)
|
||||
if !isFlushing {
|
||||
fsutils.WriterLimiter.Release()
|
||||
}
|
||||
writer, err := os.OpenFile(tmpPath, flags, 0666)
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
_ = os.MkdirAll(dir, 0777)
|
||||
|
||||
// open file again
|
||||
fsutils.WriterLimiter.Ack()
|
||||
writer, err = os.OpenFile(tmpPath, flags, 0666)
|
||||
fp, err = os.OpenFile(tmpPath, flags, 0666)
|
||||
fsutils.WriterLimiter.Release()
|
||||
}
|
||||
if err != nil {
|
||||
@@ -647,6 +648,8 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
}
|
||||
}
|
||||
|
||||
var writer = fsutils.NewFile(fp, fsutils.FlagWrite)
|
||||
|
||||
var removeOnFailure = true
|
||||
defer func() {
|
||||
if err != nil {
|
||||
@@ -663,7 +666,9 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
}()
|
||||
|
||||
// 尝试锁定,如果锁定失败,则直接返回
|
||||
fsutils.WriterLimiter.Ack()
|
||||
err = syscall.Flock(int(writer.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
removeOnFailure = false
|
||||
return nil, fmt.Errorf("%w (003)", ErrFileIsWriting)
|
||||
@@ -700,9 +705,7 @@ func (this *FileStorage) openWriter(key string, expiredAt int64, status int, hea
|
||||
metaBodySize = bodySize
|
||||
}
|
||||
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_, err = writer.Write(metaBytes)
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1001,6 +1004,11 @@ func (this *FileStorage) CanSendfile() bool {
|
||||
return this.options.EnableSendfile
|
||||
}
|
||||
|
||||
// Options 获取当前缓存存储的选项
|
||||
func (this *FileStorage) Options() *serverconfigs.HTTPFileCacheStorage {
|
||||
return this.options
|
||||
}
|
||||
|
||||
// 获取Key对应的文件路径
|
||||
func (this *FileStorage) keyPath(key string) (hash string, path string, diskIsFull bool) {
|
||||
hash = stringutil.Md5(key)
|
||||
@@ -1283,9 +1291,9 @@ func (this *FileStorage) hotLoop() {
|
||||
size = len(result) / 10
|
||||
}
|
||||
|
||||
var buf = utils.BytePool16k.Get()
|
||||
var buf = bytepool.Pool16k.Get()
|
||||
|
||||
defer utils.BytePool16k.Put(buf)
|
||||
defer bytepool.Pool16k.Put(buf)
|
||||
for _, item := range result[:size] {
|
||||
reader, err := this.openReader(item.Key, false, false, false)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,15 +3,15 @@ package caches
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/fasttime"
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
memutils "github.com/TeaOSLab/EdgeNode/internal/utils/mem"
|
||||
setutils "github.com/TeaOSLab/EdgeNode/internal/utils/sets"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/trackers"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"math"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package caches_test
|
||||
|
||||
|
||||
@@ -6,14 +6,13 @@ import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type FileWriter struct {
|
||||
storage StorageInterface
|
||||
rawWriter *os.File
|
||||
rawWriter *fsutils.File
|
||||
key string
|
||||
|
||||
metaHeaderSize int
|
||||
@@ -26,9 +25,11 @@ type FileWriter struct {
|
||||
maxSize int64
|
||||
endFunc func()
|
||||
once sync.Once
|
||||
|
||||
modifiedBytes int
|
||||
}
|
||||
|
||||
func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, maxSize int64, endFunc func()) *FileWriter {
|
||||
func NewFileWriter(storage StorageInterface, rawWriter *fsutils.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, maxSize int64, endFunc func()) *FileWriter {
|
||||
return &FileWriter{
|
||||
storage: storage,
|
||||
key: key,
|
||||
@@ -43,9 +44,7 @@ func NewFileWriter(storage StorageInterface, rawWriter *os.File, key string, exp
|
||||
|
||||
// WriteHeader 写入数据
|
||||
func (this *FileWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
n, err = this.rawWriter.Write(data)
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.headerSize += int64(n)
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
@@ -79,7 +78,7 @@ func (this *FileWriter) Write(data []byte) (n int, err error) {
|
||||
var l = len(data)
|
||||
if l > (2 << 20) {
|
||||
var offset = 0
|
||||
const bufferSize = 256 << 10
|
||||
const bufferSize = 64 << 10
|
||||
for {
|
||||
var end = offset + bufferSize
|
||||
if end > l {
|
||||
@@ -145,24 +144,18 @@ func (this *FileWriter) Close() error {
|
||||
|
||||
err := this.WriteHeaderLength(types.Int(this.headerSize))
|
||||
if err != nil {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
_ = fsutils.Remove(path)
|
||||
return err
|
||||
}
|
||||
err = this.WriteBodyLength(this.bodySize)
|
||||
if err != nil {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
_ = fsutils.Remove(path)
|
||||
return err
|
||||
}
|
||||
|
||||
fsutils.WriterLimiter.Ack()
|
||||
err = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
_ = fsutils.Remove(path)
|
||||
} else if strings.HasSuffix(path, FileTmpSuffix) {
|
||||
@@ -181,9 +174,7 @@ func (this *FileWriter) Discard() error {
|
||||
this.endFunc()
|
||||
})
|
||||
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
|
||||
err := fsutils.Remove(this.rawWriter.Name())
|
||||
return err
|
||||
@@ -211,9 +202,7 @@ func (this *FileWriter) ItemType() ItemType {
|
||||
}
|
||||
|
||||
func (this *FileWriter) write(data []byte) (n int, err error) {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
n, err = this.rawWriter.Write(data)
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.bodySize += int64(n)
|
||||
|
||||
if this.maxSize > 0 && this.bodySize > this.maxSize {
|
||||
@@ -227,5 +216,6 @@ func (this *FileWriter) write(data []byte) (n int, err error) {
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches
|
||||
|
||||
@@ -7,12 +7,12 @@ import (
|
||||
fsutils "github.com/TeaOSLab/EdgeNode/internal/utils/fs"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type PartialFileWriter struct {
|
||||
rawWriter *os.File
|
||||
rawWriter *fsutils.File
|
||||
key string
|
||||
|
||||
metaHeaderSize int
|
||||
@@ -31,9 +31,11 @@ type PartialFileWriter struct {
|
||||
|
||||
ranges *PartialRanges
|
||||
rangePath string
|
||||
|
||||
writtenBytes int64
|
||||
}
|
||||
|
||||
func NewPartialFileWriter(rawWriter *os.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
|
||||
func NewPartialFileWriter(rawWriter *fsutils.File, key string, expiredAt int64, metaHeaderSize int, metaBodySize int64, isNew bool, isPartial bool, bodyOffset int64, ranges *PartialRanges, endFunc func()) *PartialFileWriter {
|
||||
return &PartialFileWriter{
|
||||
key: key,
|
||||
rawWriter: rawWriter,
|
||||
@@ -54,9 +56,7 @@ func (this *PartialFileWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
if !this.isNew {
|
||||
return
|
||||
}
|
||||
fsutils.WriterLimiter.Ack()
|
||||
n, err = this.rawWriter.Write(data)
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.headerSize += int64(n)
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
@@ -65,9 +65,7 @@ func (this *PartialFileWriter) WriteHeader(data []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
func (this *PartialFileWriter) AppendHeader(data []byte) error {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_, err := this.rawWriter.Write(data)
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
} else {
|
||||
@@ -94,9 +92,7 @@ func (this *PartialFileWriter) WriteHeaderLength(headerLength int) error {
|
||||
_ = this.Discard()
|
||||
return err
|
||||
}
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_, err = this.rawWriter.Write(bytes4)
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
return err
|
||||
@@ -106,9 +102,7 @@ func (this *PartialFileWriter) WriteHeaderLength(headerLength int) error {
|
||||
|
||||
// Write 写入数据
|
||||
func (this *PartialFileWriter) Write(data []byte) (n int, err error) {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
n, err = this.rawWriter.Write(data)
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.bodySize += int64(n)
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
@@ -132,11 +126,14 @@ func (this *PartialFileWriter) WriteAt(offset int64, data []byte) error {
|
||||
// prevent extending too much space in a single writing
|
||||
var maxOffset = this.ranges.Max()
|
||||
if offset-maxOffset > 16<<20 {
|
||||
var extendSizePerStep int64 = 1 << 20
|
||||
var maxExtendSize int64 = 32 << 20
|
||||
if fsutils.DiskIsExtremelyFast() {
|
||||
maxExtendSize = 128 << 20
|
||||
extendSizePerStep = 4 << 20
|
||||
} else if fsutils.DiskIsFast() {
|
||||
maxExtendSize = 64 << 20
|
||||
extendSizePerStep = 2 << 20
|
||||
}
|
||||
if offset-maxOffset > maxExtendSize {
|
||||
stat, err := this.rawWriter.Stat()
|
||||
@@ -145,11 +142,8 @@ func (this *PartialFileWriter) WriteAt(offset int64, data []byte) error {
|
||||
}
|
||||
|
||||
// extend min size to prepare for file tail
|
||||
const extendSizePerStep = 8 << 20
|
||||
if stat.Size()+extendSizePerStep <= this.bodyOffset+offset+int64(len(data)) {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Truncate(stat.Size() + extendSizePerStep)
|
||||
fsutils.WriterLimiter.Release()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -163,15 +157,25 @@ func (this *PartialFileWriter) WriteAt(offset int64, data []byte) error {
|
||||
this.bodyOffset = SizeMeta + int64(keyLength) + this.headerSize
|
||||
}
|
||||
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_, err := this.rawWriter.WriteAt(data, this.bodyOffset+offset)
|
||||
fsutils.WriterLimiter.Release()
|
||||
n, err := this.rawWriter.WriteAt(data, this.bodyOffset+offset)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.ranges.Add(offset, end)
|
||||
|
||||
// 保存ranges内容到文件,当新增数据达到一定量时就更新,是为了及时更新ranges文件,以便于其他请求能够及时读取到已经缓存的部分内容
|
||||
this.writtenBytes += int64(n)
|
||||
if this.writtenBytes > (1 << 20) {
|
||||
this.writtenBytes = 0
|
||||
if len(this.rangePath) > 0 {
|
||||
if this.bodySize > 0 {
|
||||
this.ranges.BodySize = this.bodySize
|
||||
}
|
||||
_ = this.ranges.WriteToFile(this.rangePath)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -180,6 +184,14 @@ func (this *PartialFileWriter) SetBodyLength(bodyLength int64) {
|
||||
this.bodySize = bodyLength
|
||||
}
|
||||
|
||||
// SetContentMD5 设置内容MD5
|
||||
func (this *PartialFileWriter) SetContentMD5(contentMD5 string) {
|
||||
if strings.Contains(contentMD5, "\n") || len(contentMD5) > 128 {
|
||||
return
|
||||
}
|
||||
this.ranges.ContentMD5 = contentMD5
|
||||
}
|
||||
|
||||
// WriteBodyLength 写入Body长度数据
|
||||
func (this *PartialFileWriter) WriteBodyLength(bodyLength int64) error {
|
||||
if this.metaBodySize > 0 && this.metaBodySize == bodyLength {
|
||||
@@ -192,9 +204,7 @@ func (this *PartialFileWriter) WriteBodyLength(bodyLength int64) error {
|
||||
_ = this.Discard()
|
||||
return err
|
||||
}
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_, err = this.rawWriter.Write(bytes8)
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
_ = this.Discard()
|
||||
return err
|
||||
@@ -208,12 +218,12 @@ func (this *PartialFileWriter) Close() error {
|
||||
this.endFunc()
|
||||
})
|
||||
|
||||
this.ranges.BodySize = this.bodySize
|
||||
if this.bodySize > 0 {
|
||||
this.ranges.BodySize = this.bodySize
|
||||
}
|
||||
err := this.ranges.WriteToFile(this.rangePath)
|
||||
if err != nil {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.remove()
|
||||
return err
|
||||
}
|
||||
@@ -222,25 +232,19 @@ func (this *PartialFileWriter) Close() error {
|
||||
if this.isNew {
|
||||
err = this.WriteHeaderLength(types.Int(this.headerSize))
|
||||
if err != nil {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.remove()
|
||||
return err
|
||||
}
|
||||
err = this.WriteBodyLength(this.bodySize)
|
||||
if err != nil {
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
this.remove()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fsutils.WriterLimiter.Ack()
|
||||
err = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
if err != nil {
|
||||
this.remove()
|
||||
}
|
||||
@@ -254,9 +258,7 @@ func (this *PartialFileWriter) Discard() error {
|
||||
this.endFunc()
|
||||
})
|
||||
|
||||
fsutils.WriterLimiter.Ack()
|
||||
_ = this.rawWriter.Close()
|
||||
fsutils.WriterLimiter.Release()
|
||||
|
||||
SharedPartialRangesQueue.Delete(this.rangePath)
|
||||
|
||||
@@ -292,6 +294,10 @@ func (this *PartialFileWriter) IsNew() bool {
|
||||
return this.isNew && len(this.ranges.Ranges) == 0
|
||||
}
|
||||
|
||||
func (this *PartialFileWriter) Ranges() *PartialRanges {
|
||||
return this.ranges
|
||||
}
|
||||
|
||||
func (this *PartialFileWriter) remove() {
|
||||
_ = fsutils.Remove(this.rawWriter.Name())
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package caches_test
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestPartialFileWriter_Write(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ranges = caches.NewPartialRanges(0)
|
||||
var writer = caches.NewPartialFileWriter(fp, "test", time.Now().Unix()+86500, -1, -1, true, true, 0, ranges, func() {
|
||||
var writer = caches.NewPartialFileWriter(fsutils.NewFile(fp, fsutils.FlagWrite), "test", time.Now().Unix()+86500, -1, -1, true, true, 0, ranges, func() {
|
||||
t.Log("end")
|
||||
})
|
||||
_, err = writer.WriteHeader([]byte("header"))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build !plus || !linux
|
||||
|
||||
package compressions
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build !plus || !linux
|
||||
|
||||
package compressions
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package compressions_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package conns
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package conns
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build !plus
|
||||
// +build !plus
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build plus
|
||||
// +build plus
|
||||
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.3.7"
|
||||
Version = "1.3.9"
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
Role = "node"
|
||||
|
||||
EncryptKey = "8f983f4d69b83aaa0d74b21a212f6967"
|
||||
EncryptMethod = "aes-256-cfb"
|
||||
|
||||
// SystemdServiceName systemd
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2021 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package teaconst
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
package encrypt
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMagicKeyEncode(t *testing.T) {
|
||||
dst := MagicKeyEncode([]byte("Hello,World"))
|
||||
t.Log("dst:", string(dst))
|
||||
|
||||
src := MagicKeyDecode(dst)
|
||||
t.Log("src:", string(src))
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
package encrypt
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFindMethodInstance(t *testing.T) {
|
||||
t.Log(NewMethodInstance("a", "b", ""))
|
||||
t.Log(NewMethodInstance("aes-256-cfb", "123456", ""))
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package firewalls
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/zero"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/zero"
|
||||
"github.com/iwind/TeaGo/lists"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
stringutil "github.com/iwind/TeaGo/utils/string"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package firewalls
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/conns"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
|
||||
package firewalls
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package firewalls
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/firewalls/nftables"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/google/nftables/expr"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"net"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package nftables
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package nftables
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package nftables_test
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package nftables
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2023 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package nftables
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2023 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package nftables_test
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package nftables
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build linux
|
||||
|
||||
package nftables
|
||||
@@ -8,9 +8,9 @@ import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
executils "github.com/TeaOSLab/EdgeNode/internal/utils/exec"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/iwind/TeaGo/logs"
|
||||
"os"
|
||||
"runtime"
|
||||
@@ -90,18 +90,26 @@ func (this *Installer) Install() error {
|
||||
}
|
||||
|
||||
var cmd *executils.Cmd
|
||||
var aptCmd *executils.Cmd
|
||||
|
||||
// check dnf
|
||||
dnfExe, err := executils.LookPath("dnf")
|
||||
if err == nil {
|
||||
cmd = executils.NewCmd(dnfExe, "-y", "install", "nftables")
|
||||
{
|
||||
dnfExe, err := executils.LookPath("dnf")
|
||||
if err == nil {
|
||||
cmd = executils.NewCmd(dnfExe, "-y", "install", "nftables")
|
||||
}
|
||||
}
|
||||
|
||||
// check apt
|
||||
if cmd == nil {
|
||||
aptExe, err := executils.LookPath("apt")
|
||||
aptGetExe, err := executils.LookPath("apt-get")
|
||||
if err == nil {
|
||||
cmd = executils.NewCmd(aptExe, "install", "nftables")
|
||||
cmd = executils.NewCmd(aptGetExe, "install", "nftables")
|
||||
}
|
||||
|
||||
aptExe, aptErr := executils.LookPath("apt")
|
||||
if aptErr == nil {
|
||||
aptCmd = executils.NewCmd(aptExe, "install", "nftables")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,9 +127,19 @@ func (this *Installer) Install() error {
|
||||
|
||||
cmd.WithTimeout(10 * time.Minute)
|
||||
cmd.WithStderr()
|
||||
err = cmd.Run()
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s", err, cmd.Stderr())
|
||||
// try 'apt-get' instead of 'apt'
|
||||
if aptCmd != nil {
|
||||
cmd = aptCmd
|
||||
cmd.WithTimeout(10 * time.Minute)
|
||||
cmd.WithStderr()
|
||||
err = cmd.Run()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s", err, cmd.Stderr())
|
||||
}
|
||||
}
|
||||
|
||||
remotelogs.Println("NFTABLES", "installed nftables with command '"+cmd.String()+"' successfully")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
|
||||
// Copyright 2022 GoEdge goedge.cdn@gmail.com. All rights reserved.
|
||||
//go:build linux
|
||||
|
||||
package nftables
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user