根据Accept-Encoding决定是否解压响应内容

internal/caches/reader.go

@@ -24,6 +24,9 @@ type Reader interface {
 	// ReadBody 读取Body
 	ReadBody(buf []byte, callback ReaderFunc) error
 
+	// Read 实现io.Reader接口
+	Read(buf []byte) (int, error)
+
 	// ReadBodyRange 读取某个范围内的Body
 	ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error
 

internal/caches/reader_file.go

@@ -222,6 +222,37 @@ func (this *FileReader) ReadBody(buf []byte, callback ReaderFunc) error {
 	return nil
 }
 
+func (this *FileReader) Read(buf []byte) (n int, err error) {
+	var isOk = false
+
+	defer func() {
+		if !isOk {
+			_ = this.discard()
+		}
+	}()
+
+	// 直接返回从Header中剩余的
+	if this.bodyBufLen > 0 && len(buf) >= this.bodyBufLen {
+		copy(buf, this.bodyBuf)
+		isOk = true
+		n = this.bodyBufLen
+
+		if this.bodySize <= int64(this.bodyBufLen) {
+			err = io.EOF
+			return
+		}
+
+		this.bodyBufLen = 0
+		return
+	}
+
+	n, err = this.fp.Read(buf)
+	if err == nil || err == io.EOF {
+		isOk = true
+	}
+	return
+}
+
 func (this *FileReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error {
 	isOk := false
 

internal/caches/reader_memory.go

@@ -2,10 +2,13 @@ package caches
 
 import (
 	"errors"
+	"io"
 )
 
 type MemoryReader struct {
 	item *MemoryItem
+
+	offset int
 }
 
 func NewMemoryReader(item *MemoryItem) *MemoryReader {
@@ -111,6 +114,33 @@ func (this *MemoryReader) ReadBody(buf []byte, callback ReaderFunc) error {
 	return nil
 }
 
+func (this *MemoryReader) Read(buf []byte) (n int, err error) {
+	bufLen := len(buf)
+	if bufLen == 0 {
+		return 0, errors.New("using empty buffer")
+	}
+
+	bodySize := len(this.item.BodyValue)
+	left := bodySize - this.offset
+	if bufLen <= left {
+		copy(buf, this.item.BodyValue[this.offset:this.offset+bufLen])
+		n = bufLen
+
+		this.offset += bufLen
+		if this.offset >= bodySize {
+			err = io.EOF
+			return
+		}
+
+		return
+	} else {
+		copy(buf, this.item.BodyValue[this.offset:])
+		n = left
+		err = io.EOF
+		return
+	}
+}
+
 func (this *MemoryReader) ReadBodyRange(buf []byte, start int64, end int64, callback ReaderFunc) error {
 	offset := start
 	bodySize := int64(len(this.item.BodyValue))

internal/compressions/reader_brotli.go

@@ -5,6 +5,7 @@ package compressions
 import (
 	"github.com/andybalholm/brotli"
 	"io"
+	"strings"
 )
 
 type BrotliReader struct {
@@ -16,7 +17,11 @@ func NewBrotliReader(reader io.Reader) (Reader, error) {
 }
 
 func (this *BrotliReader) Read(p []byte) (n int, err error) {
-	return this.reader.Read(p)
+	n, err = this.reader.Read(p)
+	if err != nil && strings.Contains(err.Error(), "excessive") {
+		err = io.EOF
+	}
+	return
 }
 
 func (this *BrotliReader) Close() error {

internal/js/.gitignore

@@ -0,0 +1 @@
+*

internal/js/console.go

@@ -1,50 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"encoding/json"
-	"github.com/iwind/TeaGo/logs"
-	"reflect"
-)
-
-type Console struct {
-}
-
-func (this *Console) Log(args ...interface{}) {
-	for index, arg := range args {
-		if arg != nil {
-			switch arg.(type) {
-			case bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, string:
-			default:
-				var argType = reflect.TypeOf(arg)
-
-				// 是否有String()方法，如果有直接调用
-				method, ok := argType.MethodByName("String")
-				if ok && method.Type.NumIn() == 1 && method.Type.NumOut() == 1 && method.Type.Out(0).Kind() == reflect.String {
-					args[index] = method.Func.Call([]reflect.Value{reflect.ValueOf(arg)})[0].String()
-					continue
-				}
-
-				// 转为JSON
-				argJSON, err := this.toJSON(arg)
-				if err != nil {
-					if argType.Kind() == reflect.Func {
-						args[index] = "[function]"
-					} else {
-						args[index] = "[object]"
-					}
-				} else {
-					args[index] = string(argJSON)
-				}
-			}
-		} else {
-			args[index] = "null"
-		}
-	}
-	logs.Println(append([]interface{}{"[js][console]"}, args...)...)
-}
-
-func (this *Console) toJSON(o interface{}) ([]byte, error) {
-	return json.Marshal(o)
-}

internal/js/console_test.go

@@ -1,38 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"testing"
-)
-
-func TestConsole_Log(t *testing.T) {
-	{
-		vm := NewVM()
-		_, err := vm.RunString("console.log('Hello', 'world')")
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	{
-		vm := NewVM()
-		_, err := vm.RunString("console.log(null, true, false, 10, 10.123)")
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	{
-		vm := NewVM()
-		_, err := vm.RunString("console.log({ a:1, b:2 })")
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	{
-		vm := NewVM()
-		_, err := vm.RunString("console.log(console.log)")
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}

internal/js/http.go

@@ -1,36 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-type HTTP struct {
-	r RequestInterface
-
-	req  *Request
-	resp *Response
-
-	onRequest func(req *Request, resp *Response)
-}
-
-func NewHTTP(r RequestInterface) *HTTP {
-	return &HTTP{
-		req:  NewRequest(r),
-		resp: NewResponse(r),
-	}
-}
-
-func (this *HTTP) OnRequest(callback func(req *Request, resp *Response)) {
-	// TODO 考虑是否支持多个callback
-	this.onRequest = callback
-}
-
-func (this *HTTP) OnData(callback func(req *Request, resp *Response)) {
-	// TODO
-}
-
-func (this *HTTP) OnResponse(callback func(req *Request, resp *Response)) {
-	// TODO
-}
-
-func (this *HTTP) TriggerRequest() {
-	this.onRequest(this.req, this.resp)
-}

internal/js/request.go

@@ -1,82 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"bytes"
-	"io/ioutil"
-	"net"
-)
-
-type Request struct {
-	r RequestInterface
-}
-
-func NewRequest(r RequestInterface) *Request {
-	return &Request{
-		r: r,
-	}
-}
-
-func (this *Request) Proto() string {
-	return this.r.JSRequest().Proto
-}
-
-func (this *Request) Method() string {
-	return this.r.JSRequest().Method
-}
-
-func (this *Request) Header() map[string][]string {
-	return this.r.JSRequest().Header
-}
-
-func (this *Request) AddHeader(name string, value string) {
-	this.r.JSRequest().Header[name] = append(this.r.JSRequest().Header[name], value)
-}
-
-func (this *Request) SetHeader(name string, value string) {
-	this.r.JSRequest().Header[name] = []string{value}
-}
-
-func (this *Request) RemoteAddr() string {
-	var remoteAddr = this.r.JSRequest().RemoteAddr
-	host, _, err := net.SplitHostPort(remoteAddr)
-	if err == nil {
-		return host
-	}
-	return remoteAddr
-}
-
-func (this *Request) Url() *URL {
-	return NewURL(this.r.JSRequest().URL)
-}
-
-func (this *Request) ContentLength() int64 {
-	return this.r.JSRequest().ContentLength
-}
-
-func (this *Request) Body() []byte {
-	var bodyReader = this.r.JSRequest().Body
-	if bodyReader == nil {
-		return []byte{}
-	}
-	data, err := ioutil.ReadAll(bodyReader)
-	if err != nil {
-		this.r.JSLog("read body failed: " + err.Error())
-	}
-	return data
-}
-
-func (this *Request) CopyBody() []byte {
-	var bodyReader = this.r.JSRequest().Body
-	if bodyReader == nil {
-		return []byte{}
-	}
-
-	data, err := ioutil.ReadAll(bodyReader)
-	if err != nil {
-		this.r.JSLog("read body failed: " + err.Error())
-	}
-	this.r.JSRequest().Body = ioutil.NopCloser(bytes.NewReader(data))
-	return data
-}

internal/js/request_interface.go

@@ -1,19 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import "net/http"
-
-type RequestInterface interface {
-	// JSRequest 请求
-	JSRequest() *http.Request
-
-	// JSWriter 响应
-	JSWriter() http.ResponseWriter
-
-	// JSStop 中止请求
-	JSStop()
-
-	// JSLog 打印日志
-	JSLog(msg ...interface{})
-}

internal/js/request_test.go

@@ -1,124 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js_test
-
-import (
-	"bytes"
-	"github.com/TeaOSLab/EdgeNode/internal/js"
-	"github.com/iwind/TeaGo/logs"
-	"io/ioutil"
-	"net/http"
-	"testing"
-)
-
-type testRequest struct {
-	rawRequest  *http.Request
-	rawResponse *testResponse
-}
-
-func (this *testRequest) JSRequest() *http.Request {
-	if this.rawRequest != nil {
-		return this.rawRequest
-	}
-	req, _ := http.NewRequest(http.MethodGet, "https://iwind:123456@goedge.cn/docs?name=Libai&age=20", nil)
-	req.Header.Set("Server", "edgejs/1.0")
-	req.Header.Set("Content-Type", "application/json")
-	req.Body = ioutil.NopCloser(bytes.NewReader([]byte("123456")))
-	this.rawRequest = req
-	return req
-}
-
-func (this *testRequest) JSWriter() http.ResponseWriter {
-	if this.rawResponse != nil {
-		return this.rawResponse
-	}
-	this.rawResponse = &testResponse{}
-	return this.rawResponse
-}
-
-func (this *testRequest) JSStop() {
-
-}
-
-func (this *testRequest) JSLog(msg ...interface{}) {
-	logs.Println(msg...)
-}
-
-type testResponse struct {
-	statusCode int
-	header     http.Header
-}
-
-func (this *testResponse) Header() http.Header {
-	if this.header == nil {
-		this.header = http.Header{}
-	}
-	return this.header
-}
-
-func (this *testResponse) Write(p []byte) (int, error) {
-	return len(p), nil
-}
-
-func (this *testResponse) WriteHeader(statusCode int) {
-	this.statusCode = statusCode
-}
-
-func TestRequest(t *testing.T) {
-	vm := js.NewVM()
-	vm.SetRequest(&testRequest{})
-
-	// 事件监听
-	_, err := vm.RunString(`
-	http.onRequest(function (req, resp) {
-		console.log(req.proto())
-
-		let url = req.url()
-		console.log(url, "port:", url.port(), "args:", url.args())
-		console.log("username:", url.username(), "password:", url.password())
-		console.log("uri:", url.uri(), "path:", url.path())
-
-		req.addHeader("Server", "1.0")
-
-		
-		resp.write("this is response")
-		console.log(resp)
-
-		console.log(req.body()) 
-	})
-`)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// 触发事件
-	_, err = vm.RunString(`http.triggerRequest()`)
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestRequest_Header(t *testing.T) {
-	var req = js.NewRequest(&testRequest{})
-	logs.PrintAsJSON(req.Header(), t)
-
-	req.AddHeader("Content-Length", "10")
-	req.AddHeader("Vary", "1.0")
-	req.AddHeader("Vary", "2.0")
-	logs.PrintAsJSON(req.Header(), t)
-
-	req.SetHeader("Vary", "3.0")
-	logs.PrintAsJSON(req.Header(), t)
-}
-
-func TestRequest_Body(t *testing.T) {
-	var req = js.NewRequest(&testRequest{})
-	t.Log(string(req.Body()))
-	t.Log(string(req.Body()))
-}
-
-func TestRequest_CopyBody(t *testing.T) {
-	var req = js.NewRequest(&testRequest{})
-	t.Log(string(req.CopyBody()))
-	t.Log(string(req.CopyBody()))
-}

internal/js/response.go

@@ -1,39 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-type Response struct {
-	r RequestInterface
-}
-
-func NewResponse(r RequestInterface) *Response {
-	return &Response{
-		r: r,
-	}
-}
-
-func (this *Response) Write(s string) error {
-	_, err := this.r.JSWriter().Write([]byte(s))
-	return err
-}
-
-func (this *Response) Reply(status int) {
-	this.SetStatus(status)
-	this.r.JSStop()
-}
-
-func (this *Response) Header() map[string][]string {
-	return this.r.JSWriter().Header()
-}
-
-func (this *Response) AddHeader(name string, value string) {
-	this.r.JSWriter().Header()[name] = append(this.r.JSWriter().Header()[name], value)
-}
-
-func (this *Response) SetHeader(name string, value string) {
-	this.r.JSWriter().Header()[name] = []string{value}
-}
-
-func (this *Response) SetStatus(statusCode int) {
-	this.r.JSWriter().WriteHeader(statusCode)
-}

internal/js/response_test.go

@@ -1,16 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js_test
-
-import (
-	"github.com/TeaOSLab/EdgeNode/internal/js"
-	"testing"
-)
-
-func TestNewResponse(t *testing.T) {
-	var resp = js.NewResponse(&testRequest{})
-	resp.AddHeader("Vary", "1.0")
-	resp.AddHeader("Vary", "2.0")
-	resp.SetHeader("Server", "edgejs/1.0")
-	t.Logf("%#v", resp.Header())
-}

internal/js/url.go

@@ -1,90 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"github.com/dop251/goja"
-	"github.com/iwind/TeaGo/types"
-	"net/url"
-)
-
-type URL struct {
-	u *url.URL
-}
-
-func NewURL(u *url.URL) *URL {
-	return &URL{
-		u: u,
-	}
-}
-
-func (this *URL) JSNew(args []goja.Value) *URL {
-	var urlString = ""
-	if len(args) == 1 {
-		urlString = args[0].String()
-	}
-	u, _ := url.Parse(urlString)
-	if u == nil {
-		u = &url.URL{}
-	}
-	return NewURL(u)
-}
-
-func (this *URL) Port() int {
-	return types.Int(this.u.Port())
-}
-
-func (this *URL) Args() map[string][]string {
-	return this.u.Query()
-}
-
-func (this *URL) Arg(name string) string {
-	return this.u.Query().Get(name)
-}
-
-func (this *URL) Username() string {
-	if this.u.User != nil {
-		return this.u.User.Username()
-	}
-	return ""
-}
-
-func (this *URL) Password() string {
-	if this.u.User != nil {
-		password, _ := this.u.User.Password()
-		return password
-	}
-	return ""
-}
-
-func (this *URL) Uri() string {
-	return this.u.RequestURI()
-}
-
-func (this *URL) Path() string {
-	return this.u.Path
-}
-
-func (this *URL) Host() string {
-	return this.u.Host
-}
-
-func (this *URL) Fragment() string {
-	return this.u.Fragment
-}
-
-func (this *URL) Hash() string {
-	if len(this.u.Fragment) > 0 {
-		return "#" + this.u.Fragment
-	} else {
-		return ""
-	}
-}
-
-func (this *URL) Scheme() string {
-	return this.u.Scheme
-}
-
-func (this *URL) String() string {
-	return this.u.String()
-}

internal/js/url_test.go

@@ -1,18 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"net/url"
-	"testing"
-)
-
-func TestURL(t *testing.T) {
-	raw, err := url.Parse("https://iwind:123456@goedge.cn/docs?name=Libai&age=20#a=b")
-	if err != nil {
-		t.Fatal(err)
-	}
-	var u = NewURL(raw)
-	t.Log("host:", u.Host())
-	t.Log("hash:", u.Hash())
-}

internal/js/vm.go

@@ -1,153 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"errors"
-	"github.com/dop251/goja"
-	"github.com/iwind/TeaGo/logs"
-	"reflect"
-	"strings"
-)
-
-var sharedPrograms []*goja.Program
-var sharedConsole = &Console{}
-
-func init() {
-	// compile programs
-}
-
-type VM struct {
-	vm *goja.Runtime
-}
-
-func NewVM() *VM {
-	vm := goja.New()
-	vm.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))
-
-	// programs
-	for _, program := range sharedPrograms {
-		_, _ = vm.RunProgram(program)
-	}
-
-	v := &VM{vm: vm}
-	v.initVM()
-	return v
-}
-
-func (this *VM) Set(name string, obj interface{}) error {
-	return this.vm.Set(name, obj)
-}
-
-func (this *VM) AddConstructor(name string, instance interface{}) error {
-	objType := reflect.TypeOf(instance)
-
-	if objType.Kind() != reflect.Ptr {
-		return errors.New("instance should be pointer")
-	}
-
-	// construct
-	newMethod, ok := objType.MethodByName("JSNew")
-	if !ok {
-		return errors.New("can not find 'JSNew()' method in '" + objType.Elem().Name() + "'")
-	}
-
-	var err = this.Set(name, func(call goja.ConstructorCall) *goja.Object {
-		if newMethod.Type.NumIn() != 2 {
-			this.throw(errors.New(objType.Elem().Name() + ".JSNew() should accept a '[]goja.Value' argument"))
-			return nil
-		}
-		if newMethod.Type.In(1).String() != "[]goja.Value" {
-			this.throw(errors.New(objType.Elem().Name() + ".JSNew() should accept a '[]goja.Value' argument"))
-			return nil
-		}
-
-		// new
-		var results = newMethod.Func.Call([]reflect.Value{reflect.ValueOf(instance), reflect.ValueOf(call.Arguments)})
-		if len(results) == 0 {
-			this.throw(errors.New(objType.Elem().Name() + ".JSNew() should return a valid instance"))
-			return nil
-		}
-		var result = results[0]
-		if result.Type() != objType {
-			this.throw(errors.New(objType.Elem().Name() + ".JSNew() should return a same instance"))
-			return nil
-		}
-
-		// methods
-		var resultType = result.Type()
-		var numMethod = result.NumMethod()
-		for i := 0; i < numMethod; i++ {
-			var method = resultType.Method(i)
-			var methodName = strings.ToLower(method.Name[:1]) + method.Name[1:]
-			err := call.This.Set(methodName, result.MethodByName(method.Name).Interface())
-			if err != nil {
-				this.throw(err)
-				continue
-			}
-		}
-
-		//  支持属性
-		var numField = result.Elem().Type().NumField()
-		for i := 0; i < numField; i++ {
-			var field = result.Elem().Field(i)
-			if !field.CanInterface() {
-				continue
-			}
-			var fieldType = objType.Elem().Field(i)
-			tag, ok := fieldType.Tag.Lookup("json")
-			if !ok {
-				tag = fieldType.Name
-				tag = strings.ToLower(tag[:1]) + tag[1:]
-			} else {
-				// TODO 校验tag是否符合变量语法
-			}
-			err := call.This.Set(tag, field.Interface())
-			if err != nil {
-				this.throw(err)
-				continue
-			}
-		}
-
-		return nil
-	})
-	return err
-}
-
-func (this *VM) RunString(str string) (goja.Value, error) {
-	defer func() {
-		e := recover()
-		if e != nil {
-			// TODO 需要打印trace
-			logs.Println("panic:", e)
-		}
-	}()
-	return this.vm.RunString(str)
-}
-
-func (this *VM) SetRequest(req RequestInterface) {
-	{
-		err := this.vm.Set("http", NewHTTP(req))
-		if err != nil {
-			this.throw(err)
-		}
-	}
-}
-
-func (this *VM) initVM() {
-	{
-		err := this.vm.Set("console", sharedConsole)
-		if err != nil {
-			this.throw(err)
-		}
-	}
-}
-
-func (this *VM) throw(err error) {
-	if err == nil {
-		return
-	}
-
-	// TODO
-	logs.Println("js:VM:error: " + err.Error())
-}

internal/js/vm_test.go

@@ -1,158 +0,0 @@
-// Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
-
-package js
-
-import (
-	"github.com/dop251/goja"
-	"testing"
-	"time"
-)
-
-func TestNewVM(t *testing.T) {
-	before := time.Now()
-	defer func() {
-		t.Log(time.Since(before).Seconds()*1000, "ms")
-	}()
-
-	vm := NewVM()
-	{
-		v, err := vm.RunString("JSON.stringify({\"a\":\"b\"})")
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("JSON.stringify():", v)
-	}
-	{
-		v, err := vm.RunString(`JSON.parse('{\"a\":\"b\"}')`)
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Log("JSON.parse():", v)
-	}
-	{
-		err := vm.AddConstructor("Url", &URL{})
-		if err != nil {
-			t.Fatal("add constructor error:", err)
-		}
-		_, err = vm.RunString(`
-{
-	let u = new Url("https://goedge.cn/docs?v=1")
-	console.log("host:", u.host(), u.uri())
-}
-{
-	let u = new Url("https://teaos.cn/downloads?v=1")
-	console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url()
-	console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url("a", "b", "c")
-	console.log("host:", u.host(), u.uri())
-}
-`)
-		if err != nil {
-			t.Fatal("add constructor error:" + err.Error())
-		}
-	}
-}
-
-func TestVM_Program(t *testing.T) {
-	var s = `
-{
-	let u = new Url("https://goedge.cn/docs?v=1")
-	//console.log("host:", u.host(), u.uri())
-}
-{
-	let u = new Url("https://teaos.cn/downloads?v=1")
-	//console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url()
-	//console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url("a", "b", "c")
-	//console.log("host:", u.host(), u.uri())
-}
-`
-	program := goja.MustCompile("s", s, true)
-
-	before := time.Now()
-	defer func() {
-		t.Log(time.Since(before).Seconds()*1000, "ms")
-	}()
-
-	vm := NewVM()
-	err := vm.AddConstructor("Url", &URL{})
-	if err != nil {
-		t.Fatal("add constructor error:", err)
-	}
-	//_, err = vm.RunString(s)
-	_, err = vm.vm.RunProgram(program)
-	if err != nil {
-		t.Fatal("add constructor error:" + err.Error())
-	}
-}
-
-func Benchmark_Program(b *testing.B) {
-	var s = `
-{
-	let u = new Url("https://goedge.cn/docs?v=1")
-	//console.log("host:", u.host(), u.uri())
-}
-{
-	let u = new Url("https://teaos.cn/downloads?v=1")
-	//console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url()
-	//console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url("a", "b", "c")
-	//console.log("host:", u.host(), u.uri())
-}
-{
-	let u = new Url("https://goedge.cn/docs?v=1")
-	//console.log("host:", u.host(), u.uri())
-}
-{
-	let u = new Url("https://teaos.cn/downloads?v=1")
-	//console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url()
-	//console.log("host:", u.host(), u.uri())
-}
-
-{
-	let u = new Url("a", "b", "c")
-	//console.log("host:", u.host(), u.uri())
-}
-`
-	program := goja.MustCompile("s", s, true)
-
-	vm := NewVM()
-
-	err := vm.AddConstructor("Url", &URL{})
-	if err != nil {
-		b.Fatal("add constructor error:", err)
-	}
-
-	for i := 0; i < b.N; i++ {
-		//_, err = vm.RunString(s)
-		_, err = vm.vm.RunProgram(program)
-		if err != nil {
-			b.Fatal("add constructor error:" + err.Error())
-		}
-	}
-}

internal/nodes/client_conn.go

@@ -39,7 +39,7 @@ func NewClientConn(conn net.Conn, isTLS bool, quickClose bool, globalLimiter *ra
 func (this *ClientConn) Read(b []byte) (n int, err error) {
 	if this.isTLS {
 		if !this.hasRead {
-			_ = this.rawConn.SetReadDeadline(time.Now().Add(5 * time.Second)) // TODO 握手超时时间可以设置
+			_ = this.rawConn.SetReadDeadline(time.Now().Add(time.Duration(nodeconfigs.DefaultTLSHandshakeTimeout) * time.Second)) // TODO 握手超时时间可以设置
 			this.hasRead = true
 			defer func() {
 				_ = this.rawConn.SetReadDeadline(time.Time{})
@@ -48,7 +48,6 @@ func (this *ClientConn) Read(b []byte) (n int, err error) {
 	}
 
 	n, err = this.rawConn.Read(b)
-
 	if n > 0 {
 		atomic.AddUint64(&teaconst.InTrafficBytes, uint64(n))
 	}
@@ -66,6 +65,8 @@ func (this *ClientConn) Write(b []byte) (n int, err error) {
 func (this *ClientConn) Close() error {
 	this.isClosed = true
 
+	err := this.rawConn.Close()
+
 	// 全局并发数限制
 	this.once.Do(func() {
 		if this.globalLimiter != nil {
@@ -76,7 +77,7 @@ func (this *ClientConn) Close() error {
 	// 单个服务并发数限制
 	sharedClientConnLimiter.Remove(this.rawConn.RemoteAddr().String())
 
-	return this.rawConn.Close()
+	return err
 }
 
 func (this *ClientConn) LocalAddr() net.Addr {

internal/nodes/client_listener.go

@@ -20,7 +20,7 @@ type ClientListener struct {
 	quickClose  bool
 }
 
-func NewClientListener1(listener net.Listener, quickClose bool) *ClientListener {
+func NewClientListener(listener net.Listener, quickClose bool) *ClientListener {
 	return &ClientListener{
 		rawListener: listener,
 		quickClose:  quickClose,

internal/nodes/http_request_cache.go

@@ -5,10 +5,12 @@ import (
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
 	"github.com/TeaOSLab/EdgeNode/internal/caches"
+	"github.com/TeaOSLab/EdgeNode/internal/compressions"
 	"github.com/TeaOSLab/EdgeNode/internal/goman"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/rpc"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
+	"io"
 	"net/http"
 	"path/filepath"
 	"strconv"
@@ -431,16 +433,28 @@ func (this *HTTPRequest) doCacheRead(useStale bool) (shouldStop bool) {
 				return true
 			}
 		} else { // 没有Range
+			var body io.Reader = reader
+			var contentEncoding = this.writer.Header().Get("Content-Encoding")
+			if len(contentEncoding) > 0 && !httpAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"), contentEncoding) {
+				decompressReader, err := compressions.NewReader(body, contentEncoding)
+				if err == nil {
+					body = decompressReader
+					defer func() {
+						_ = decompressReader.Close()
+					}()
+
+					this.writer.Header().Del("Content-Encoding")
+					this.writer.Header().Del("Content-Length")
+				}
+			}
+
 			this.writer.PrepareCompression(reader.BodySize())
 			this.writer.WriteHeader(reader.Status())
 
-			err = reader.ReadBody(buf, func(n int) (goNext bool, err error) {
-				_, err = this.writer.Write(buf[:n])
-				if err != nil {
-					return false, errWritingToClient
-				}
-				return true, nil
-			})
+			_, err = io.CopyBuffer(this.writer, body, buf)
+			if err == io.EOF {
+				err = nil
+			}
 			if err != nil {
 				this.varMapping["cache.status"] = "MISS"
 

internal/nodes/http_request_reverse_proxy.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs"
 	"github.com/TeaOSLab/EdgeCommon/pkg/serverconfigs/shared"
+	"github.com/TeaOSLab/EdgeNode/internal/compressions"
 	"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
 	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"io"
@@ -253,6 +254,24 @@ func (this *HTTPRequest) doReverseProxy() {
 		}
 	}
 
+	// 解压
+	if !resp.Uncompressed {
+		var contentEncoding = resp.Header.Get("Content-Encoding")
+		if len(contentEncoding) > 0 && !httpAcceptEncoding(this.RawReq.Header.Get("Accept-Encoding"), contentEncoding) {
+			reader, err := compressions.NewReader(resp.Body, contentEncoding)
+			if err == nil {
+				var body = resp.Body
+				defer func() {
+					_ = body.Close()
+				}()
+
+				resp.Body = reader
+				resp.Header.Del("Content-Encoding")
+				resp.Header.Del("Content-Length")
+			}
+		}
+	}
+
 	// 响应Header
 	this.writer.AddHeaders(resp.Header)
 	this.processResponseHeaders(resp.StatusCode)

internal/nodes/http_request_utils.go

@@ -153,3 +153,19 @@ func httpRequestNextId() string {
 	// timestamp + requestId + nodeId
 	return strconv.FormatInt(unixTime, 10) + teaconst.NodeIdString + strconv.Itoa(int(atomic.AddInt32(&httpRequestId, 1)))
 }
+
+// 检查是否可以接受某个编码
+func httpAcceptEncoding(acceptEncodings string, encoding string) bool {
+	var pieces = strings.Split(acceptEncodings, ",")
+	for _, piece := range pieces {
+		var qualityIndex = strings.Index(piece, ";")
+		if qualityIndex >= 0 {
+			piece = piece[:qualityIndex]
+		}
+
+		if strings.TrimSpace(piece) == encoding {
+			return true
+		}
+	}
+	return false
+}

internal/nodes/listener.go

@@ -60,7 +60,7 @@ func (this *Listener) listenTCP() error {
 	if err != nil {
 		return err
 	}
-	var netListener = NewClientListener1(tcpListener, protocol.IsHTTPFamily() || protocol.IsHTTPSFamily())
+	var netListener = NewClientListener(tcpListener, protocol.IsHTTPFamily() || protocol.IsHTTPSFamily())
 	events.On(events.EventQuit, func() {
 		remotelogs.Println("LISTENER", "quit "+this.group.FullAddr())
 		_ = netListener.Close()

internal/stats/http_request_stat_manager_test.go

@@ -14,9 +14,9 @@ func TestHTTPRequestStatManager_Loop_Region(t *testing.T) {
 	iplibrary.SharedLibrary = library
 
 	manager := NewHTTPRequestStatManager()
-	manager.AddRemoteAddr(11, "202.196.0.20")
-	manager.AddRemoteAddr(11, "202.196.0.20") // 重复添加一个测试相加
-	manager.AddRemoteAddr(11, "8.8.8.8")
+	manager.AddRemoteAddr(11, "202.196.0.20", 0, false)
+	manager.AddRemoteAddr(11, "202.196.0.20", 0, false) // 重复添加一个测试相加
+	manager.AddRemoteAddr(11, "8.8.8.8", 0, false)
 
 	/**for i := 0; i < 100; i++ {
 		manager.AddRemoteAddr(11, strconv.Itoa(rands.Int(10, 250))+"."+strconv.Itoa(rands.Int(10, 250))+"."+strconv.Itoa(rands.Int(10, 250))+".8")

internal/stats/traffic_stat_manager_test.go

@@ -8,7 +8,7 @@ import (
 func TestTrafficStatManager_Add(t *testing.T) {
 	manager := NewTrafficStatManager()
 	for i := 0; i < 100; i++ {
-		manager.Add(1, "goedge.cn", 1, 0, 0, 0, 0, 0)
+		manager.Add(1, "goedge.cn", 1, 0, 0, 0, 0, 0, false, 0)
 	}
 	t.Log(manager.itemMap)
 }
@@ -16,7 +16,7 @@ func TestTrafficStatManager_Add(t *testing.T) {
 func TestTrafficStatManager_Upload(t *testing.T) {
 	manager := NewTrafficStatManager()
 	for i := 0; i < 100; i++ {
-		manager.Add(1, "goedge.cn", 1, 0, 0, 0, 0, 0)
+		manager.Add(1, "goedge.cn", 1, 0, 0, 0, 0, 0, false, 0)
 	}
 	err := manager.Upload()
 	if err != nil {
@@ -30,6 +30,6 @@ func BenchmarkTrafficStatManager_Add(b *testing.B) {
 
 	manager := NewTrafficStatManager()
 	for i := 0; i < b.N; i++ {
-		manager.Add(1, "goedge.cn", 1024, 1, 0, 0, 0, 0)
+		manager.Add(1, "goedge.cn", 1024, 1, 0, 0, 0, 0, false, 0)
 	}
 }

internal/ttlcache/cache.go

@@ -1,6 +1,7 @@
 package ttlcache
 
 import (
+	"github.com/TeaOSLab/EdgeNode/internal/utils"
 	"time"
 )
 
@@ -21,12 +22,18 @@ type Cache struct {
 }
 
 func NewCache(opt ...OptionInterface) *Cache {
-	countPieces := 128
-	maxItems := 2_000_000
+	var countPieces = 128
+	var maxItems = 2_000_000
 
-	var delta = systemMemoryGB() / 8
-	if delta > 0 {
-		maxItems *= delta
+	var totalMemory = utils.SystemMemoryGB()
+	if totalMemory < 2 {
+		// 我们限制内存过小的服务能够使用的数量
+		maxItems = 1_000_000
+	} else {
+		var delta = totalMemory / 8
+		if delta > 0 {
+			maxItems *= delta
+		}
 	}
 
 	for _, option := range opt {

internal/ttlcache/cache_test.go

@@ -29,12 +29,19 @@ func TestNewCache(t *testing.T) {
 }
 
 func TestCache_Memory(t *testing.T) {
+	var stat1 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat1)
+
 	cache := NewCache()
-	for i := 0; i < 20_000_000; i++ {
+	for i := 0; i < 10_000_000; i++ {
 		cache.Write("a"+strconv.Itoa(i), 1, time.Now().Unix()+3600)
 	}
 	t.Log("waiting ...")
-	time.Sleep(10 * time.Second)
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+
+	t.Log((stat2.HeapInuse-stat1.HeapInuse)/1024/1024, "MB")
 }
 
 func BenchmarkCache_Add(b *testing.B) {

internal/ttlcache/piece.go

@@ -20,8 +20,14 @@ func NewPiece(maxItems int) *Piece {
 func (this *Piece) Add(key uint64, item *Item) (ok bool) {
 	this.locker.Lock()
 	if len(this.m) >= this.maxItems {
-		this.locker.Unlock()
-		return
+		// 尝试先删除过期的
+		this.gcWithoutLocker()
+
+		// 仍然是满的就跳过
+		if len(this.m) >= this.maxItems {
+			this.locker.Unlock()
+			return
+		}
 	}
 	this.m[key] = item
 	this.locker.Unlock()
@@ -74,12 +80,7 @@ func (this *Piece) Count() (count int) {
 
 func (this *Piece) GC() {
 	this.locker.Lock()
-	timestamp := time.Now().Unix()
-	for k, item := range this.m {
-		if item.expiredAt <= timestamp {
-			delete(this.m, k)
-		}
-	}
+	this.gcWithoutLocker()
 	this.locker.Unlock()
 }
 
@@ -94,3 +95,13 @@ func (this *Piece) Destroy() {
 	this.m = nil
 	this.locker.Unlock()
 }
+
+// 不加锁的gc
+func (this *Piece) gcWithoutLocker() {
+	timestamp := time.Now().Unix()
+	for k, item := range this.m {
+		if item.expiredAt <= timestamp {
+			delete(this.m, k)
+		}
+	}
+}

internal/utils/system.go

@@ -1,6 +1,6 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package ttlcache
+package utils
 
 import (
 	"github.com/shirou/gopsutil/mem"
@@ -8,7 +8,11 @@ import (
 
 var systemTotalMemory = -1
 
-func systemMemoryGB() int {
+func init() {
+	_ = SystemMemoryGB()
+}
+
+func SystemMemoryGB() int {
 	if systemTotalMemory > 0 {
 		return systemTotalMemory
 	}

internal/utils/system_test.go

@@ -1,11 +1,11 @@
 // Copyright 2021 Liuxiangchao iwind.liu@gmail.com. All rights reserved.
 
-package ttlcache
+package utils
 
 import "testing"
 
 func TestSystemMemoryGB(t *testing.T) {
-	t.Log(systemMemoryGB())
-	t.Log(systemMemoryGB())
-	t.Log(systemMemoryGB())
+	t.Log(SystemMemoryGB())
+	t.Log(SystemMemoryGB())
+	t.Log(SystemMemoryGB())
 }