新版IP库管理阶段性提交（未完成）

2022-08-13 23:55:59 +08:00
parent 507457dec1
commit 4ddc1ac89f
42 changed files with 10035 additions and 3582 deletions
@@ -0,0 +1 @@
+     ip.db
@@ -0,0 +1,13 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+type ipItem struct {
+	ipFrom     uint64
+	ipTo       uint64
+	countryId  int64
+	provinceId int64
+	cityId     int64
+	townId     int64
+	providerId int64
+}
@@ -0,0 +1,39 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+type Meta struct {
+	Version   int         `json:"version"` // IP库版本
+	Author    string      `json:"author"`
+	Countries []*Country  `json:"countries"`
+	Provinces []*Province `json:"provinces"`
+	Cities    []*City     `json:"cities"`
+	Towns     []*Town     `json:"towns"`
+	Providers []*Provider `json:"providers"`
+	CreatedAt int64       `json:"createdAt"`
+}
+
+type Provider struct {
+	Name  string   `json:"name"`
+	Codes []string `json:"codes"`
+}
+
+type Country struct {
+	Name  string   `json:"name"`
+	Codes []string `json:"codes"`
+}
+
+type Province struct {
+	Name  string   `json:"name"`
+	Codes []string `json:"codes"`
+}
+
+type City struct {
+	Name  string   `json:"name"`
+	Codes []string `json:"codes"`
+}
+
+type Town struct {
+	Name  string   `json:"name"`
+	Codes []string `json:"codes"`
+}
@@ -0,0 +1,64 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"bytes"
+	"errors"
+)
+
+type Parser struct {
+	config *ParserConfig
+
+	data []byte
+}
+
+func NewParser(config *ParserConfig) (*Parser, error) {
+	if config == nil {
+		config = &ParserConfig{}
+	}
+
+	if config.Template == nil {
+		return nil, errors.New("template must not be nil")
+	}
+
+	return &Parser{
+		config: config,
+	}, nil
+}
+
+func (this *Parser) Write(data []byte) {
+	this.data = append(this.data, data...)
+}
+
+func (this *Parser) Parse() error {
+	if len(this.data) == 0 {
+		return nil
+	}
+	for {
+		var index = bytes.IndexByte(this.data, '\n')
+		if index >= 0 {
+			var line = this.data[:index+1]
+			values, found := this.config.Template.Extract(string(line), this.config.EmptyValues)
+			if found {
+				if this.config.Iterator != nil {
+					err := this.config.Iterator(values)
+					if err != nil {
+						return err
+					}
+				}
+			} else {
+				// 防止错误信息太长
+				if len(line) > 256 {
+					line = line[:256]
+				}
+				return errors.New("invalid line '" + string(line) + "'")
+			}
+
+			this.data = this.data[index+1:]
+		} else {
+			break
+		}
+	}
+	return nil
+}
@@ -0,0 +1,9 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+type ParserConfig struct {
+	Template    *Template
+	EmptyValues []string
+	Iterator    func(values map[string]string) error
+}
@@ -0,0 +1,54 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"errors"
+	"io"
+)
+
+type ReaderParser struct {
+	reader    io.Reader
+	rawParser *Parser
+}
+
+func NewReaderParser(reader io.Reader, config *ParserConfig) (*ReaderParser, error) {
+	if config == nil {
+		config = &ParserConfig{}
+	}
+
+	if config.Template == nil {
+		return nil, errors.New("template must not be nil")
+	}
+
+	parser, err := NewParser(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ReaderParser{
+		reader:    reader,
+		rawParser: parser,
+	}, nil
+}
+
+func (this *ReaderParser) Parse() error {
+	var buf = make([]byte, 1024)
+	for {
+		n, err := this.reader.Read(buf)
+		if n > 0 {
+			this.rawParser.Write(buf[:n])
+			parseErr := this.rawParser.Parse()
+			if parseErr != nil {
+				return parseErr
+			}
+		}
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+			break
+		}
+	}
+	return nil
+}
@@ -0,0 +1,62 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"bytes"
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"testing"
+)
+
+func TestNewReaderParser(t *testing.T) {
+	template, err := iplibrary.NewTemplate("${ipFrom}|${ipTo}|${country}|${any}|${province}|${city}|${provider}")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var buf = &bytes.Buffer{}
+	buf.WriteString(`8.45.160.0|8.45.161.255|美国|0|华盛顿|西雅图|Level3
+8.45.162.0|8.45.162.255|美国|0|马萨诸塞|0|Level3
+8.45.163.0|8.45.164.255|美国|0|俄勒冈|0|Level3
+8.45.165.0|8.45.165.255|美国|0|华盛顿|0|Level3
+8.45.166.0|8.45.167.255|美国|0|华盛顿|西雅图|Level3
+8.45.168.0|8.127.255.255|美国|0|0|0|Level3
+8.128.0.0|8.128.3.255|中国|0|上海|上海市|阿里巴巴
+8.128.4.0|8.128.255.255|中国|0|0|0|阿里巴巴
+8.129.0.0|8.129.255.255|中国|0|广东省|深圳市|阿里云
+8.130.0.0|8.130.55.255|中国|0|北京|北京市|阿里云
+8.130.56.0|8.131.255.255|中国|0|北京|北京市|阿里巴巴
+8.132.0.0|8.133.255.255|中国|0|上海|上海市|阿里巴巴
+8.134.0.0|8.134.20.63|中国|0|0|0|阿里云
+8.134.20.64|8.134.79.255|中国|0|广东省|深圳市|阿里云
+8.134.80.0|8.191.255.255|中国|0|0|0|阿里巴巴
+8.192.0.0|8.192.0.255|美国|0|0|0|Level3
+8.192.1.0|8.192.1.255|美国|0|马萨诸塞|波士顿|Level3
+8.192.2.0|8.207.255.255|美国|0|0|0|Level3
+8.208.0.0|8.208.127.255|英国|0|伦敦|伦敦|阿里云
+8.208.128.0|8.208.255.255|英国|0|伦敦|伦敦|阿里巴巴
+8.209.0.0|8.209.15.255|俄罗斯|0|莫斯科|莫斯科|阿里云
+8.209.16.0|8.209.31.255|俄罗斯|0|莫斯科|莫斯科|阿里巴巴
+8.209.32.0|8.209.32.15|中国|0|台湾省|台北|阿里云
+8.209.32.16|8.209.32.255|中国|0|台湾省|台北|阿里巴巴
+8.209.33.0|8.209.34.255|中国|0|台湾省|台北|阿里云
+8.209.35.0|8.209.35.255|中国|0|台湾省|台北|阿里巴巴`)
+
+	var count int
+	parser, err := iplibrary.NewReaderParser(buf, &iplibrary.ParserConfig{
+		Template:    template,
+		EmptyValues: []string{"0"},
+		Iterator: func(values map[string]string) error {
+			count++
+			t.Log(count, values)
+			return nil
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = parser.Parse()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
@@ -0,0 +1,44 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"testing"
+)
+
+func TestNewParser(t *testing.T) {
+	template, err := iplibrary.NewTemplate("${ipFrom}|${ipTo}|${country}|${any}|${province}|${city}|${provider}")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	parser, err := iplibrary.NewParser(&iplibrary.ParserConfig{
+		Template:    template,
+		EmptyValues: []string{"0"},
+		Iterator: func(values map[string]string) error {
+			t.Log(values)
+			return nil
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	parser.Write([]byte(`0.0.0.0|0.255.255.255|0|0|0|内网IP|内网IP
+1.0.0.0|1.0.0.255|澳大利亚|0|0|0|0
+1.0.1.0|1.0.3.255|中国|0|福建省|福州市|电信
+1.0.4.0|1.0.7.255|澳大利亚|0|维多利亚|墨尔本|0
+1.0.8.0|1.0.15.255|中国|0|广东省|广州市|电信
+1.0.16.0|1.0.31.255|日本|0|0|0|0
+1.0.32.0|1.0.63.255|中国|0|广东省|广州市|电信
+1.0.64.0|1.0.79.255|日本|0|广岛县|0|0
+1.0.80.0|1.0.127.255|日本|0|冈山县|0|0
+1.0.128.0|1.0.128.255|泰国|0|清莱府|0|TOT
+1.0.129.0|1.0.132.191|泰国|0|曼谷|曼谷|TOT`))
+
+	err = parser.Parse()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
@@ -0,0 +1,211 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/iwind/TeaGo/types"
+	"io"
+	"net"
+	"sort"
+	"strings"
+)
+
+type Reader struct {
+	meta *Meta
+
+	ipV4Items []*ipItem
+	ipV6Items []*ipItem
+}
+
+func NewReader(reader io.Reader) (*Reader, error) {
+	var libReader = &Reader{}
+	err := libReader.load(reader)
+	if err != nil {
+		return nil, err
+	}
+	return libReader, nil
+}
+
+func (this *Reader) load(reader io.Reader) error {
+	var buf = make([]byte, 1024)
+	var metaLine = []byte{}
+	var metaLineFound = false
+	var dataBuf = []byte{}
+	for {
+		n, err := reader.Read(buf)
+		if n > 0 {
+			var data = buf[:n]
+			dataBuf = append(dataBuf, data...)
+			if metaLineFound {
+				left, err := this.parse(dataBuf)
+				if err != nil {
+					return err
+				}
+				dataBuf = left
+			} else {
+				var index = bytes.IndexByte(dataBuf, '\n')
+				if index > 0 {
+					metaLine = dataBuf[:index]
+					dataBuf = dataBuf[index+1:]
+					metaLineFound = true
+					var meta = &Meta{}
+					err = json.Unmarshal(metaLine, &meta)
+					if err != nil {
+						return err
+					}
+					this.meta = meta
+
+					left, err := this.parse(dataBuf)
+					if err != nil {
+						return err
+					}
+					dataBuf = left
+				}
+			}
+		}
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+			break
+		}
+	}
+
+	sort.Slice(this.ipV4Items, func(i, j int) bool {
+		var from0 = this.ipV4Items[i].ipFrom
+		var to0 = this.ipV4Items[i].ipTo
+		var from1 = this.ipV4Items[j].ipFrom
+		var to1 = this.ipV4Items[j].ipTo
+		if from0 == from1 {
+			return to0 < to1
+		}
+		return from0 < from1
+	})
+
+	sort.Slice(this.ipV6Items, func(i, j int) bool {
+		var from0 = this.ipV6Items[i].ipFrom
+		var to0 = this.ipV6Items[i].ipTo
+		var from1 = this.ipV6Items[j].ipFrom
+		var to1 = this.ipV6Items[j].ipTo
+		if from0 == from1 {
+			return to0 < to1
+		}
+		return from0 < from1
+	})
+
+	return nil
+}
+
+func (this *Reader) Lookup(ip net.IP) *QueryResult {
+	var ipLong = configutils.IP2Long(ip)
+	var isV4 = configutils.IsIPv4(ip)
+	var resultItem *ipItem
+	if isV4 {
+		sort.Search(len(this.ipV4Items), func(i int) bool {
+			var item = this.ipV4Items[i]
+			if item.ipFrom <= ipLong {
+				if item.ipTo >= ipLong {
+					resultItem = item
+					return false
+				}
+				return false
+			}
+			return true
+		})
+	} else {
+		sort.Search(len(this.ipV6Items), func(i int) bool {
+			var item = this.ipV6Items[i]
+			if item.ipFrom <= ipLong {
+				if item.ipTo >= ipLong {
+					resultItem = item
+					return false
+				}
+				return false
+			}
+			return true
+		})
+	}
+
+	return &QueryResult{
+		item: resultItem,
+	}
+}
+
+func (this *Reader) Meta() *Meta {
+	return this.meta
+}
+
+func (this *Reader) IPv4Items() []*ipItem {
+	return this.ipV4Items
+}
+
+func (this *Reader) IPv6Items() []*ipItem {
+	return this.ipV6Items
+}
+
+func (this *Reader) parse(data []byte) (left []byte, err error) {
+	if len(data) == 0 {
+		return
+	}
+	for {
+		var index = bytes.IndexByte(data, '\n')
+		if index >= 0 {
+			var line = data[:index]
+			var pieces = strings.Split(string(line), "|")
+			if len(pieces) != 8 {
+				return nil, errors.New("invalid ip definition '" + string(line) + "'")
+			}
+
+			var version = pieces[0]
+			if len(version) == 0 {
+				version = "4"
+			}
+
+			if version != "4" && version != "6" {
+				return nil, errors.New("invalid ip version '" + string(line) + "'")
+			}
+
+			if len(pieces[2]) == 0 {
+				pieces[2] = pieces[1]
+			}
+
+			var ipFrom = types.Uint64(pieces[1])
+			var ipTo = types.Uint64(pieces[2])
+			if ipFrom > ipTo {
+				ipFrom, ipTo = ipTo, ipFrom
+			}
+
+			if version == "4" {
+				this.ipV4Items = append(this.ipV4Items, &ipItem{
+					ipFrom:     ipFrom,
+					ipTo:       ipTo,
+					countryId:  types.Int64(pieces[3]),
+					provinceId: types.Int64(pieces[4]),
+					cityId:     types.Int64(pieces[5]),
+					townId:     types.Int64(pieces[6]),
+					providerId: types.Int64(pieces[7]),
+				})
+			} else {
+				this.ipV6Items = append(this.ipV6Items, &ipItem{
+					ipFrom:     ipFrom,
+					ipTo:       ipTo,
+					countryId:  types.Int64(pieces[3]),
+					provinceId: types.Int64(pieces[4]),
+					cityId:     types.Int64(pieces[5]),
+					townId:     types.Int64(pieces[6]),
+					providerId: types.Int64(pieces[7]),
+				})
+			}
+
+			data = data[index+1:]
+		} else {
+			left = data
+			break
+		}
+	}
+	return
+}
@@ -0,0 +1,41 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"compress/gzip"
+	"net"
+	"os"
+)
+
+type FileReader struct {
+	rawReader *Reader
+}
+
+func NewFileReader(path string) (*FileReader, error) {
+	fp, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = fp.Close()
+	}()
+
+	gzReader, err := gzip.NewReader(fp)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := NewReader(gzReader)
+	if err != nil {
+		return nil, err
+	}
+
+	return &FileReader{
+		rawReader: reader,
+	}, nil
+}
+
+func (this *FileReader) Lookup(ip net.IP) *QueryResult {
+	return this.rawReader.Lookup(ip)
+}
@@ -0,0 +1,37 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"net"
+	"testing"
+)
+
+func TestNewFileReader(t *testing.T) {
+	reader, err := iplibrary.NewFileReader("./ip.db")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, ip := range []string{
+		"127.0.0.1",
+		"192.168.0.1",
+		"192.168.0.150",
+		"192.168.1.100",
+		"192.168.2.100",
+		"192.168.3.50",
+		"192.168.0.150",
+		"192.168.4.80",
+		"8.8.8.8",
+		"111.197.165.199",
+		"175.178.206.125",
+	} {
+		var result = reader.Lookup(net.ParseIP(ip))
+		if result.IsOk() {
+			t.Log(ip+":", "countryId:", result.CountryId(), "provinceId:", result.ProvinceId(), "cityId:", result.CityId(), "provider:", result.ProviderId())
+		} else {
+			t.Log(ip+":", "not found")
+		}
+	}
+}
@@ -0,0 +1,46 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+type QueryResult struct {
+	item *ipItem
+}
+
+func (this *QueryResult) IsOk() bool {
+	return this.item != nil
+}
+
+func (this *QueryResult) CountryId() int64 {
+	if this.item != nil {
+		return this.item.countryId
+	}
+	return 0
+}
+
+func (this *QueryResult) ProvinceId() int64 {
+	if this.item != nil {
+		return this.item.provinceId
+	}
+	return 0
+}
+
+func (this *QueryResult) CityId() int64 {
+	if this.item != nil {
+		return this.item.cityId
+	}
+	return 0
+}
+
+func (this *QueryResult) TownId() int64 {
+	if this.item != nil {
+		return this.item.townId
+	}
+	return 0
+}
+
+func (this *QueryResult) ProviderId() int64 {
+	if this.item != nil {
+		return this.item.providerId
+	}
+	return 0
+}
@@ -0,0 +1,149 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"bytes"
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	timeutil "github.com/iwind/TeaGo/utils/time"
+	"net"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func TestNewReader(t *testing.T) {
+	var buf = &bytes.Buffer{}
+	var writer = iplibrary.NewWriter(buf, &iplibrary.Meta{
+		Author: "GoEdge <https://goedge.cn>",
+	})
+
+	err := writer.WriteMeta()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.1.100", "192.168.1.100", 1, 200, 300, 400, 500)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.2.100", "192.168.3.100", 2, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.3.101", "192.168.3.101", 3, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.0.101", "192.168.0.200", 4, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("::1", "::5", 5, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	/**var n = func() string {
+		return types.String(rands.Int(0, 255))
+	}
+
+	for i := 0; i < 1_000_000; i++ {
+		err = writer.Write(n()+"."+n()+"."+n()+"."+n(), n()+"."+n()+"."+n()+"."+n(), int64(i)+100, 201, 301, 401, 501)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}**/
+
+	var stat = &runtime.MemStats{}
+	runtime.ReadMemStats(stat)
+	reader, err := iplibrary.NewReader(buf)
+
+	var stat2 = &runtime.MemStats{}
+	runtime.ReadMemStats(stat2)
+	t.Log((stat2.Alloc-stat.Alloc)/1024/1024, "M")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("version:", reader.Meta().Version, "author:", reader.Meta().Author, "createdTime:", timeutil.FormatTime("Y-m-d H:i:s", reader.Meta().CreatedAt))
+
+	if len(reader.IPv4Items()) < 10 {
+		t.Log("===")
+		for _, item := range reader.IPv4Items() {
+			t.Logf("%+v", item)
+		}
+		t.Log("===")
+	}
+	if len(reader.IPv6Items()) < 10 {
+		t.Log("===")
+		for _, item := range reader.IPv6Items() {
+			t.Logf("%+v", item)
+		}
+		t.Log("===")
+	}
+
+	var before = time.Now()
+	for _, ip := range []string{
+		"192.168.0.1",
+		"192.168.0.150",
+		"192.168.1.100",
+		"192.168.2.100",
+		"192.168.3.50",
+		"192.168.0.150",
+		"192.168.4.80",
+		"::3",
+		"::8",
+	} {
+		var result = reader.Lookup(net.ParseIP(ip))
+		if result.IsOk() {
+			t.Log(ip+":", "countryId:", result.CountryId())
+		} else {
+			t.Log(ip+":", "not found")
+		}
+	}
+	t.Log(time.Since(before).Seconds()*1000, "ms")
+}
+
+func BenchmarkNewReader(b *testing.B) {
+	runtime.GOMAXPROCS(1)
+
+	var buf = &bytes.Buffer{}
+	var writer = iplibrary.NewWriter(buf, &iplibrary.Meta{
+		Author: "GoEdge <https://goedge.cn>",
+	})
+
+	err := writer.WriteMeta()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var n = func() string {
+		return types.String(rands.Int(0, 255))
+	}
+
+	for i := 0; i < 1_000_000; i++ {
+		err = writer.Write(n()+"."+n()+"."+n()+"."+n(), n()+"."+n()+"."+n()+"."+n(), int64(i)+100, 201, 301, 401, 501)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	reader, err := iplibrary.NewReader(buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		var ip = "192.168.1.100"
+		reader.Lookup(net.ParseIP(ip))
+	}
+}
@@ -0,0 +1,75 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"github.com/iwind/TeaGo/lists"
+	"regexp"
+)
+
+type Template struct {
+	templateString string
+	reg            *regexp.Regexp
+}
+
+func NewTemplate(templateString string) (*Template, error) {
+	var t = &Template{
+		templateString: templateString,
+	}
+	err := t.init()
+	if err != nil {
+		return nil, err
+	}
+	return t, nil
+}
+
+func (this *Template) init() error {
+	var template = regexp.QuoteMeta(this.templateString)
+	var keywordReg = regexp.MustCompile(`\\\$\\{(\w+)\\}`)
+	template = keywordReg.ReplaceAllStringFunc(template, func(keyword string) string {
+		var matches = keywordReg.FindStringSubmatch(keyword)
+		if len(matches) > 1 {
+			switch matches[1] {
+			case "ipFrom", "ipTo", "country", "province", "city", "town", "provider":
+				return "(?P<" + matches[1] + ">.*)"
+			}
+			return ".*"
+		}
+
+		return keyword
+	})
+	reg, err := regexp.Compile(template)
+	if err != nil {
+		return err
+	}
+	this.reg = reg
+	return nil
+}
+
+func (this *Template) Extract(text string, emptyValues []string) (values map[string]string, ok bool) {
+	var matches = this.reg.FindStringSubmatch(text)
+	if len(matches) == 0 {
+		return
+	}
+	values = map[string]string{}
+	for index, name := range this.reg.SubexpNames() {
+		if len(name) == 0 {
+			continue
+		}
+		var v = matches[index]
+		if name != "ipFrom" && name != "ipTo" && (v == "0" || v == "无" || lists.ContainsString(emptyValues, v)) {
+			v = ""
+		}
+		values[name] = v
+	}
+
+	for _, keyword := range []string{"ipFrom", "ipTo", "country", "province", "city", "town", "provider"} {
+		_, hasKeyword := values[keyword]
+		if !hasKeyword {
+			values[keyword] = ""
+		}
+	}
+
+	ok = true
+	return
+}
@@ -0,0 +1,25 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"testing"
+)
+
+func TestNewTemplate(t *testing.T) {
+	template, err := iplibrary.NewTemplate("${ipFrom}|${ipTo}|${country}|${any}|${province}|${city}|${provider}")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, s := range []string{
+		"42.0.32.0|42.0.63.255|中国|0|广东省|广州市|电信",
+		"42.0.32.0|42.0.63.255|中国|0|广东省|广州市|电信\n123",
+		"42.0.32.0|42.0.63.255|中国||广东省|广州市|电信",
+		"42.0.32.0|42.0.63.255|中国|0||广州市|电信",
+		"42.0.32.0|42.0.63.255|中国|0|广东省|广州市",
+	} {
+		values, _ := template.Extract(s, []string{})
+		t.Log(s, "=>\n", values)
+	}
+}
@@ -0,0 +1,9 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+type Version = int
+
+const (
+	Version1 Version = 1
+)
@@ -0,0 +1,154 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"crypto/md5"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/TeaOSLab/EdgeCommon/pkg/configutils"
+	"github.com/iwind/TeaGo/types"
+	"hash"
+	"io"
+	"net"
+	"strings"
+	"time"
+)
+
+type hashWriter struct {
+	rawWriter io.Writer
+	hash      hash.Hash
+}
+
+func newHashWriter(writer io.Writer) *hashWriter {
+	return &hashWriter{
+		rawWriter: writer,
+		hash:      md5.New(),
+	}
+}
+
+func (this *hashWriter) Write(p []byte) (n int, err error) {
+	n, err = this.rawWriter.Write(p)
+	this.hash.Write(p)
+	return
+}
+
+func (this *hashWriter) Sum() string {
+	return fmt.Sprintf("%x", this.hash.Sum(nil))
+}
+
+type Writer struct {
+	writer *hashWriter
+	meta   *Meta
+}
+
+func NewWriter(writer io.Writer, meta *Meta) *Writer {
+	if meta == nil {
+		meta = &Meta{}
+	}
+	meta.Version = Version1
+	meta.CreatedAt = time.Now().Unix()
+
+	var libWriter = &Writer{
+		writer: newHashWriter(writer),
+		meta:   meta,
+	}
+	return libWriter
+}
+
+func (this *Writer) WriteMeta() error {
+	metaJSON, err := json.Marshal(this.meta)
+	if err != nil {
+		return err
+	}
+	_, err = this.writer.Write(metaJSON)
+	if err != nil {
+		return err
+	}
+	_, err = this.writer.Write([]byte("\n"))
+	return err
+}
+
+func (this *Writer) Write(ipFrom string, ipTo string, countryId int64, provinceId int64, cityId int64, townId int64, providerId int64) error {
+	// validate IP
+	var fromIP = net.ParseIP(ipFrom)
+	if fromIP == nil {
+		return errors.New("invalid 'ipFrom': '" + ipFrom + "'")
+	}
+	var fromIsIPv4 = configutils.IsIPv4(fromIP)
+	var toIP = net.ParseIP(ipTo)
+	if toIP == nil {
+		return errors.New("invalid 'ipTo': " + ipTo)
+	}
+	var toIsIPv4 = configutils.IsIPv4(toIP)
+	if fromIsIPv4 != toIsIPv4 {
+		return errors.New("'ipFrom(" + ipFrom + ")' and 'ipTo(" + ipTo + ")' should have the same IP version")
+	}
+
+	var pieces = []string{}
+
+	// 0
+	if fromIsIPv4 {
+		pieces = append(pieces, "")
+	} else {
+		pieces = append(pieces, "6")
+	}
+
+	// 1
+	pieces = append(pieces, types.String(configutils.IP2Long(fromIP)))
+	if ipFrom == ipTo {
+		// 2
+		pieces = append(pieces, "")
+	} else {
+		// 2
+		pieces = append(pieces, types.String(configutils.IP2Long(toIP)))
+	}
+
+	// 3
+	if countryId > 0 {
+		pieces = append(pieces, types.String(countryId))
+	} else {
+		pieces = append(pieces, "")
+	}
+
+	// 4
+	if provinceId > 0 {
+		pieces = append(pieces, types.String(provinceId))
+	} else {
+		pieces = append(pieces, "")
+	}
+
+	// 5
+	if cityId > 0 {
+		pieces = append(pieces, types.String(cityId))
+	} else {
+		pieces = append(pieces, "")
+	}
+
+	// 6
+	if townId > 0 {
+		pieces = append(pieces, types.String(townId))
+	} else {
+		pieces = append(pieces, "")
+	}
+
+	// 7
+	if providerId > 0 {
+		pieces = append(pieces, types.String(providerId))
+	} else {
+		pieces = append(pieces, "")
+	}
+
+	_, err := this.writer.Write([]byte(strings.Join(pieces, "|")))
+	if err != nil {
+		return err
+	}
+
+	_, err = this.writer.Write([]byte("\n"))
+	return err
+}
+
+func (this *Writer) Sum() string {
+	return this.writer.Sum()
+}
@@ -0,0 +1,58 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary
+
+import (
+	"compress/gzip"
+	"os"
+)
+
+type FileWriter struct {
+	fp       *os.File
+	gzWriter *gzip.Writer
+
+	rawWriter *Writer
+}
+
+func NewFileWriter(path string, meta *Meta) (*FileWriter, error) {
+	fp, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	gzWriter, err := gzip.NewWriterLevel(fp, gzip.BestCompression)
+	if err != nil {
+		return nil, err
+	}
+
+	var writer = &FileWriter{
+		fp:        fp,
+		gzWriter:  gzWriter,
+		rawWriter: NewWriter(gzWriter, meta),
+	}
+	return writer, nil
+}
+
+func (this *FileWriter) WriteMeta() error {
+	return this.rawWriter.WriteMeta()
+}
+
+func (this *FileWriter) Write(ipFrom string, ipTo string, countryId int64, provinceId int64, cityId int64, townId int64, providerId int64) error {
+	return this.rawWriter.Write(ipFrom, ipTo, countryId, provinceId, cityId, townId, providerId)
+}
+
+func (this *FileWriter) Sum() string {
+	return this.rawWriter.Sum()
+}
+
+func (this *FileWriter) Close() error {
+	err1 := this.gzWriter.Close()
+	err2 := this.fp.Close()
+	if err1 != nil {
+		return err1
+	}
+	if err2 != nil {
+		return err2
+	}
+	return nil
+}
@@ -0,0 +1,56 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"github.com/iwind/TeaGo/rands"
+	"github.com/iwind/TeaGo/types"
+	"testing"
+)
+
+func TestNewFileWriter(t *testing.T) {
+	writer, err := iplibrary.NewFileWriter("./ip.db", &iplibrary.Meta{
+		Author: "GoEdge",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.WriteMeta()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.1.100", "192.168.1.100", 100, 200, 300, 400, 500)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.2.100", "192.168.3.100", 101, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.3.101", "192.168.3.101", 101, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var n = func() string {
+		return types.String(rands.Int(0, 255))
+	}
+
+	for i := 0; i < 1_000_000; i++ {
+		err = writer.Write(n()+"."+n()+"."+n()+"."+n(), n()+"."+n()+"."+n()+"."+n(), int64(i)+100, 201, 301, 401, 501)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("ok", writer.Sum())
+}
@@ -0,0 +1,44 @@
+// Copyright 2022 Liuxiangchao iwind.liu@gmail.com. All rights reserved. Official site: https://goedge.cn .
+
+package iplibrary_test
+
+import (
+	"bytes"
+	"github.com/TeaOSLab/EdgeCommon/pkg/iplibrary"
+	"testing"
+)
+
+func TestNewWriter(t *testing.T) {
+	var buf = &bytes.Buffer{}
+	var writer = iplibrary.NewWriter(buf, &iplibrary.Meta{
+		Author: "GoEdge <https://goedge.cn>",
+	})
+
+	err := writer.WriteMeta()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.1.100", "192.168.1.100", 100, 200, 300, 400, 500)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.2.100", "192.168.3.100", 101, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("192.168.3.101", "192.168.3.101", 101, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = writer.Write("::1", "::2", 101, 201, 301, 401, 501)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log(buf.String())
+	t.Log("sum:", writer.Sum())
+}