Compare commits
210 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
2955431edb | |
|
|
c0d85cc71d | |
|
|
d5e3616d8e | |
|
|
cd40e3a1d5 | |
|
|
ee19ac95ca | |
|
|
a99989343e | |
|
|
324bfdf35d | |
|
|
f4d81dd3dd | |
|
|
ba96deaecf | |
|
|
c4eb1a14ce | |
|
|
dbef88e337 | |
|
|
30ed0cf028 | |
|
|
d06690f651 | |
|
|
daf6749c34 | |
|
|
e50ead2cba | |
|
|
b1eb00e25e | |
|
|
fac3f73193 | |
|
|
d1dda93abd | |
|
|
44013251b6 | |
|
|
546ce589a9 | |
|
|
507b7bddc1 | |
|
|
25513cb246 | |
|
|
445eb4cb8e | |
|
|
a1667d1182 | |
|
|
d189535bb8 | |
|
|
06d54d8084 | |
|
|
8849cc9130 | |
|
|
c911ca1290 | |
|
|
241831a6f0 | |
|
|
2f0dc5f33f | |
|
|
c1db56c369 | |
|
|
211c374ee2 | |
|
|
a3b8d817ee | |
|
|
b7bb22589f | |
|
|
1068bf05a3 | |
|
|
fc46254325 | |
|
|
7b320846c9 | |
|
|
e360f3d031 | |
|
|
e7c240a8d9 | |
|
|
d638015592 | |
|
|
3eb65a2657 | |
|
|
fac3d0376a | |
|
|
a8b9d542c6 | |
|
|
82a73fa60d | |
|
|
8b79b69524 | |
|
|
c5fb71facb | |
|
|
62e1f98497 | |
|
|
2af5ee176c | |
|
|
0ce74b8873 | |
|
|
f4fcd5e900 | |
|
|
072b6fed24 | |
|
|
f939363a19 | |
|
|
353524325a | |
|
|
59cf547107 | |
|
|
3147144a7f | |
|
|
d4299a1d1b | |
|
|
18ecd7ae85 | |
|
|
5fbdef93cd | |
|
|
a90191e8d8 | |
|
|
593e028677 | |
|
|
72c657c378 | |
|
|
19a8816827 | |
|
|
787bd7ae07 | |
|
|
5b10989ac2 | |
|
|
90e827633a | |
|
|
4dd0a1e006 | |
|
|
77134c4315 | |
|
|
fedf5aaf03 | |
|
|
d28a10b0c1 | |
|
|
aab908c923 | |
|
|
c314ef54ca | |
|
|
69d267d85d | |
|
|
a603a75ab7 | |
|
|
d730db816a | |
|
|
d36edc6e15 | |
|
|
1d9ffabfeb | |
|
|
03cc433717 | |
|
|
3562b7af8e | |
|
|
165746ba38 | |
|
|
79243eb0ba | |
|
|
be71b92f20 | |
|
|
d380827871 | |
|
|
1a08f5e568 | |
|
|
36c9de10bb | |
|
|
61ef15cff0 | |
|
|
d6444e740f | |
|
|
e3094c8ef2 | |
|
|
5244ebad08 | |
|
|
4d391acac2 | |
|
|
5cb17713a7 | |
|
|
e1f0d5126a | |
|
|
d9fc17727e | |
|
|
dabe7798f6 | |
|
|
8f841e6237 | |
|
|
d7b60f41e4 | |
|
|
83d6fafaa0 | |
|
|
02f6b31894 | |
|
|
97e647234a | |
|
|
1209b72452 | |
|
|
62aa42e05e | |
|
|
e76f995ceb | |
|
|
edb29aea3b | |
|
|
8a7b088216 | |
|
|
897ee5c0a9 | |
|
|
1eba8ece3b | |
|
|
007cb0808a | |
|
|
7d6ea00303 | |
|
|
e183607da5 | |
|
|
b0466348da | |
|
|
04e51e2182 | |
|
|
c63b1dc5db | |
|
|
1a29562c2d | |
|
|
29c7048f93 | |
|
|
fa16a4a45d | |
|
|
8adec5a3c8 | |
|
|
51e194a105 | |
|
|
b965ab8be9 | |
|
|
a720267842 | |
|
|
112dfb99b5 | |
|
|
ca665cfd2e | |
|
|
0ad8031754 | |
|
|
fb0efface4 | |
|
|
e373f92312 | |
|
|
3d21a8247c | |
|
|
d5e10e1383 | |
|
|
1a8d01d967 | |
|
|
e87f580510 | |
|
|
962335954a | |
|
|
2c01e01d2c | |
|
|
79886f77e7 | |
|
|
132221b494 | |
|
|
aef4f91348 | |
|
|
9d4b165409 | |
|
|
4b55d14fca | |
|
|
b030661865 | |
|
|
537e6f9306 | |
|
|
aaad26244d | |
|
|
a31a966546 | |
|
|
832e3fd77d | |
|
|
ef51b901ee | |
|
|
e8242a24ea | |
|
|
30e5f2732d | |
|
|
1c6d299efa | |
|
|
dce0bb582f | |
|
|
b3115ba7aa | |
|
|
81885a16cf | |
|
|
2b61de20d6 | |
|
|
7a1970729a | |
|
|
5a53add68b | |
|
|
c9e8acaffc | |
|
|
ce92aabc11 | |
|
|
3f0480be1f | |
|
|
a694cfa8d2 | |
|
|
8a44c98f99 | |
|
|
cab8fbc4ab | |
|
|
7d11be2ce7 | |
|
|
8302525ae8 | |
|
|
e926501591 | |
|
|
87418d879a | |
|
|
16fa5572f2 | |
|
|
39c297ad04 | |
|
|
6081603bd8 | |
|
|
2861abcb83 | |
|
|
daf54bf877 | |
|
|
6c98b8faf9 | |
|
|
1dc0003661 | |
|
|
c4a3a2e1f8 | |
|
|
0e828bba0d | |
|
|
b0ab8a0b2f | |
|
|
9caef15329 | |
|
|
572fda5910 | |
|
|
67d709ab39 | |
|
|
076f3918fa | |
|
|
9411b7f8c2 | |
|
|
75c375d146 | |
|
|
5037816f41 | |
|
|
aafee317c8 | |
|
|
839d5f59a5 | |
|
|
d4180c0f57 | |
|
|
8da3f55092 | |
|
|
7ff7e5f61f | |
|
|
44fd27f027 | |
|
|
3539f5e4e3 | |
|
|
7bb61ec59c | |
|
|
89852a70f0 | |
|
|
d427b4dcd8 | |
|
|
ad4253dbb0 | |
|
|
e6e3087144 | |
|
|
ad0cfd6bd9 | |
|
|
0cf4cf8e26 | |
|
|
ba2c9ec119 | |
|
|
91f281d53c | |
|
|
05158e0a2e | |
|
|
384567f184 | |
|
|
937c0772ec | |
|
|
ebc6253dfe | |
|
|
969a5f0e03 | |
|
|
c2bb78aecd | |
|
|
e139731d9e | |
|
|
96717ab155 | |
|
|
0520f299ee | |
|
|
bbe2dddecf | |
|
|
14753ffc29 | |
|
|
3127ac0cdc | |
|
|
108ef78a51 | |
|
|
f2c988287c | |
|
|
7bab822a49 | |
|
|
1df8b55789 | |
|
|
68cd1053d2 | |
|
|
398149f17e |
|
|
@ -29,10 +29,6 @@ META-INF/
|
||||||
.vscode
|
.vscode
|
||||||
|
|
||||||
# binding
|
# binding
|
||||||
/v1.0//binding/java/classes/
|
|
||||||
/v1.0//binding/java/doc/
|
|
||||||
/v1.0//binding/java/target/
|
|
||||||
/v1.0//binding/java/*.jar
|
|
||||||
/binding/java/classes/
|
/binding/java/classes/
|
||||||
/binding/java/doc/
|
/binding/java/doc/
|
||||||
/binding/java/target/
|
/binding/java/target/
|
||||||
|
|
@ -61,18 +57,12 @@ target
|
||||||
|
|
||||||
# VS ignore cases
|
# VS ignore cases
|
||||||
/**/*.sln
|
/**/*.sln
|
||||||
/v1.0/binding/c#/**/.vs/
|
|
||||||
/v1.0/binding/c#/**/packages
|
|
||||||
/v1.0/binding/c#/**/bin
|
|
||||||
/v1.0/binding/c#/**/obj
|
|
||||||
/binding/c#/**/.vs/
|
/binding/c#/**/.vs/
|
||||||
/binding/c#/**/packages
|
/binding/c#/**/packages
|
||||||
/binding/c#/**/bin
|
/binding/c#/**/bin
|
||||||
/binding/c#/**/obj
|
/binding/c#/**/obj
|
||||||
|
|
||||||
# Nodejs
|
# Nodejs
|
||||||
/v1.0/binding/nodejs/tests/unitTests/__snapshots__
|
|
||||||
/v1.0/binding/nodejs/coverage
|
|
||||||
/binding/nodejs/tests/unitTests/__snapshots__
|
/binding/nodejs/tests/unitTests/__snapshots__
|
||||||
/binding/nodejs/coverage
|
/binding/nodejs/coverage
|
||||||
/binding/nodejs/node_modules
|
/binding/nodejs/node_modules
|
||||||
|
|
@ -99,3 +89,6 @@ target
|
||||||
#vscode
|
#vscode
|
||||||
.vscode
|
.vscode
|
||||||
build
|
build
|
||||||
|
|
||||||
|
# git worktrees
|
||||||
|
.worktrees/
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,11 @@
|
||||||
|
ip2region is dual-licensed under the Apache License 2.0 and the MIT License.
|
||||||
|
|
||||||
|
You may use, distribute, and modify this software under either license,
|
||||||
|
at your option. The full text of both licenses appears below.
|
||||||
|
|
||||||
|
SPDX-License-Identifier: Apache-2.0 OR MIT
|
||||||
|
|
||||||
|
==============================================================================
|
||||||
Apache License
|
Apache License
|
||||||
Version 2.0, January 2004
|
Version 2.0, January 2004
|
||||||
http://www.apache.org/licenses/
|
http://www.apache.org/licenses/
|
||||||
|
|
|
||||||
176
README.md
176
README.md
|
|
@ -1,113 +1,125 @@
|
||||||
# Ip2region 是什么
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
[ip2region](https://ip2region.net) - 是一个离线IP地址定位库和IP定位数据管理框架,同时支持`IPv4`和`IPv6`,10微秒级别的查询效率,提供了众多主流编程语言的 `xdb` 数据生成和查询客户端实现。
|
# ip2region
|
||||||
|
|
||||||
|
[ip2region](https://ip2region.net) - is an offline IP address localization library and IP localization data management framework. It supports both `IPv4` and `IPv6` with query efficiency at the 10-microsecond level. It provides `xdb` data generation and query client implementations for many mainstream programming languages.
|
||||||
|
|
||||||
|
|
||||||
|
# Features
|
||||||
|
|
||||||
# Ip2region 特性
|
### 1. Offline Localization Library
|
||||||
|
|
||||||
### 1、离线定位库
|
The project itself provides raw data for both IPv4 (`data/ipv4_source.txt`) and IPv6 (`data/ipv6_source.txt`), along with corresponding xdb files (`data/ip2region_v4.xdb` and `data/ip2region_v6.xdb`) to achieve city-level query localization. The field format is: `Country|Province|City|ISP|iso-alpha2-code`. Localization information for China is entirely in Chinese, while regional information for non-China areas is entirely in English.
|
||||||
|
|
||||||
项目本身同时了提供了一份 IPv4(`data/ipv4_source.txt`) 和 IPv6(`data/ipv6_source.txt`) 的原始数据和对应的 xdb 文件(`data/ip2region_v4.xdb` 和 `data/ip2region_v6.xdb`) 用于实现精确到城市的的查询定位功能。
|
### 2. Data Management Framework
|
||||||
|
|
||||||
### 2、数据管理框架
|
`xdb` supports hundreds of millions of IP data segment rows. Region information supports full customization. The region information of the built-in data is fixed in the format: `Country|Province|City|ISP|iso-alpha2-Code`. You can append data for specific business needs to the region, such as: GPS information/International Standard Regional Codes/Zip codes, etc. In other words, you can fully use ip2region to manage your own IP localization data.
|
||||||
|
|
||||||
`xdb` 支持亿级别的 IP 数据段行数,region 信息支持完全自定义,自带数据的 region 信息固定了格式为:`国家|省份|城市|ISP`,你可以在 region 中追加特定业务需求的数据,例如:GPS信息/国际统一地域信息编码/邮编等。也就是你完全可以使用 ip2region 来管理你自己的 IP 定位数据。
|
### 3. Data Deduplication and Compression
|
||||||
|
|
||||||
### 3、数据去重和压缩
|
The `xdb` format generation program automatically processes the input raw data, checks and completes the merging of adjacent IP segments, and performs deduplication and compression of identical regional information.
|
||||||
|
|
||||||
`xdb` 格式生成程序会自动处理输入的原始数据,检查并且完成相连 IP 段的的合并以及相同地域信息的去重和压缩。
|
### 4. High-Speed Query Response
|
||||||
|
|
||||||
### 4、极速查询响应
|
Even for queries based entirely on the `xdb` file, the single query response time is at the 10-microsecond level. Memory-accelerated queries can be enabled through the following two methods:
|
||||||
|
|
||||||
即使是完全基于 `xdb` 文件的查询,单次查询响应时间在十微秒级别,可通过如下两种方式开启内存加速查询:
|
1. `vIndex` Index Caching: Uses a fixed `512KiB` of memory to cache vector index data, reducing one disk IO operation and maintaining average query efficiency within 100 microseconds.
|
||||||
|
2. Entire `xdb` File Caching: Loads the entire `xdb` file into memory. Memory usage is equal to the `xdb` file size. There is no disk IO operation, maintaining 10-microsecond level query efficiency.
|
||||||
|
|
||||||
1. `vIndex` 索引缓存 :使用固定的 `512KiB` 的内存空间缓存 vector index 数据,减少一次 IO 磁盘操作,保持平均查询效率稳定在100微秒之内。
|
### 5. Unified Query Interface
|
||||||
2. `xdb` 整个文件缓存:将整个 `xdb` 文件全部加载到内存,内存占用等同于 `xdb` 文件大小,无磁盘 IO 操作,保持10微秒级别的查询效率。
|
|
||||||
|
|
||||||
### 5、统一的查询接口
|
`xdb` provides version-compatible query implementations. A unified API can simultaneously provide queries for both IPv4 and IPv6 data and return unified data.
|
||||||
|
|
||||||
`xdb` 提供了版本兼容的查询实现,一个统一的 API 可以同时提供对 IPv4 和 IPv6 数据的查询并且返回统一的数据。
|
|
||||||
|
|
||||||
# `xdb` 数据查询
|
|
||||||
|
|
||||||
API 介绍,使用文档和测试程序请参考对应 `searcher` 查询客户端下的 ReadMe 介绍,全部查询 binding 实现情况如下:
|
|
||||||
| 编程语言 | 描述 | IPv4 支持 | IPv6 支持 | 贡献者 |
|
|
||||||
|:---------------------------------|:-----------------------------------------------|:-------------------|:------------------ |:--------------------------------------------------|
|
|
||||||
| [Golang](binding/golang) | golang xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [PHP](binding/php) | php xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [Java](binding/java) | java xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [C](binding/c) | POSIX C xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [Lua_c](binding/lua_c) | lua c 扩展 xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [Lua](binding/lua) | lua xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [Rust](binding/rust) | rust xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [gongzhengyang](https://github.com/gongzhengyang) |
|
|
||||||
| [Python](binding/python) | python xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [Javascript](binding/javascript) | javascript xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
|
||||||
| [Csharp](binding/csharp) | csharp xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Alen Lee](https://github.com/malus2077) & [ArgoZhang](https://github.com/ArgoZhang) |
|
|
||||||
| [Erlang](binding/erlang) | erlang xdb 查询客户端 | :white_check_mark: | :x: | [malou](https://github.com/malou996) |
|
|
||||||
| [Nginx](binding/nginx) | nginx 扩展 xdb 查询客户端 | :white_check_mark: | :x: | [Wu Jian Ping](https://github.com/wujjpp) |
|
|
||||||
| [C++](binding/cpp) | C++ xdb 查询客户端 | :white_check_mark: | :white_check_mark: | [Yunbin Liu](https://github.com/liuyunbin) |
|
|
||||||
|
|
||||||
|
|
||||||
以下工具链实现由社区开发者通过第三方仓库贡献:
|
# `xdb` Query
|
||||||
| 编程语言 | 描述 | 贡献者 |
|
|
||||||
|:--------------------------------------------------------------- |:------------------------|:-----------------------------------------------|
|
For API introductions, usage documentation, and test programs, please refer to the README introduction under the corresponding `searcher` query client. All query binding implementations are as follows:
|
||||||
| [ip2region-composer](https://github.com/zoujingli/ip2region) | php composer 管理客户端 | [邹景立](https://github.com/zoujingli) |
|
|
||||||
| [ip2region-ts](https://github.com/Steven-Qiang/ts-ip2region2) | node.js addon 管理客户端| [Steven Qiang](https://github.com/Steven-Qiang)|
|
| Language | Description | IPv4 Support | IPv6 Support |
|
||||||
| [ruby-ip2region](https://github.com/jicheng1014/ruby-ip2region) | ruby xdb 查询客户端实现 | [jicheng1014](https://github.com/jicheng1014) |
|
| --- | --- | --- | --- |
|
||||||
| [Ip2regionTool](https://github.com/orestonce/Ip2regionTool) | ip2region 数据转换工具 | [orestonce](https://github.com/orestonce) |
|
| [Golang](binding/golang/README.md) | golang query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [PHP](binding/php/README.md) | php query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Java](binding/java/README.md) | java query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C](binding/c/README.md) | C[std=c99] query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Lua_c](binding/lua_c/README.md) | lua c extension query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Lua](binding/lua/README.md) | lua query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Rust](binding/rust/README.md) | rust query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Python](binding/python/README.md) | python query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Javascript](binding/javascript/README.md) | javascript query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Csharp](binding/csharp) | csharp query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Erlang](binding/erlang/README.md) | erlang query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Nginx](binding/nginx) | nginx extension query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C++](binding/cpp/README.md) | C++ query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Cangjie](binding/cangjie/README.md) | Cangjie query client | :white_check_mark: | :white_check_mark: |
|
||||||
|
|
||||||
|
The following toolchain implementations are contributed by community developers via third-party repositories:
|
||||||
|
|
||||||
|
| Language | Description |
|
||||||
|
| --- | --- |
|
||||||
|
| [ip2region-composer](https://github.com/zoujingli/ip2region) | php composer management client |
|
||||||
|
| [ip2region-ts](https://github.com/Steven-Qiang/ts-ip2region2) | node.js addon management client |
|
||||||
|
| [ruby-ip2region](https://github.com/jicheng1014/ruby-ip2region) | ruby xdb query client implementation |
|
||||||
|
| [Ip2regionTool](https://github.com/orestonce/Ip2regionTool) | ip2region data conversion tool |
|
||||||
|
|
||||||
|
|
||||||
# `xdb` 数据生成
|
# `xdb` Generation
|
||||||
|
|
||||||
API 介绍,使用文档和测试程序请参考如下 `maker` 生成程序下的 ReadMe 文档:
|
For API introductions, usage documentation, and test programs, please refer to the README documents under the following `maker` generation programs:
|
||||||
|
|
||||||
| 编程语言 | 描述 | IPv4 支持 | IPv6 支持 | 贡献者 |
|
| Language | Description | IPv4 Support | IPv6 Support |
|
||||||
| :---------------------- | :------------------- |:-------------------|:-------------------| :------------------------------------------------------------------------------------------ |
|
| --- | --- | --- | --- |
|
||||||
| [Golang](maker/golang) | golang xdb 生成程序 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
| [Golang](maker/golang/README.md) | golang xdb generation program | :white_check_mark: | :white_check_mark: |
|
||||||
| [Java](maker/java) | java xdb 生成程序 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
| [Java](maker/java/README.md) | java xdb generation program | :white_check_mark: | :white_check_mark: |
|
||||||
| [Python](maker/python) | python xdb 生成程序 | :white_check_mark: | :x: | [leolin49](https://github.com/leolin49) |
|
| [Python](maker/python/README.md) | python xdb generation program | :white_check_mark: | :x: |
|
||||||
| [Csharp](maker/csharp) | csharp xdb 生成程序 | :white_check_mark: | :x: | [Alan Lee](https://github.com/malus2077) |
|
| [Csharp](maker/csharp/README.md) | csharp xdb generation program | :white_check_mark: | :x: |
|
||||||
| [Rust](maker/rust) | rust xdb 生成程序 | :white_check_mark: | :white_check_mark: | [KevinWang](https://github.com/KevinWL) & [gongzhengyang](https://github.com/gongzhengyang) |
|
| [Rust](maker/rust/README.md) | rust xdb generation program | :white_check_mark: | :white_check_mark: |
|
||||||
| [C++](maker/cpp) | C++ xdb 生成程序 | :white_check_mark: | :white_check_mark: | [Yunbin Liu](https://github.com/liuyunbin) |
|
| [C++](maker/cpp) | C++ xdb generation program | :white_check_mark: | :white_check_mark: |
|
||||||
|
|
||||||
|
|
||||||
# `xdb` 数据更新
|
# `xdb` Update
|
||||||
|
|
||||||
ip2region 项目的核心在于 <b>研究 IP 数据的存储和快速查询的设计和实现</b>, 项目自带的 `./data/ipv4_source.txt` 和 `./data/ipv6_source.txt` 原始数据不会再提供更新,对于数据精度和更新频率要求很高的使用场景建议到 [Ip2Region社区](https://ip2region.net/products/offline) 或者第三方购买商用离线数据,你可以使用如下几种方式来尝试自己更新数据:
|
The core of the ip2region project lies in **researching the design and implementation of IP data storage and fast querying**. The raw data `./data/ipv4_source.txt` and `./data/ipv6_source.txt` included in the project are updated irregularly. For scenarios with high requirements for data accuracy and update frequency, it is recommended to purchase commercial offline data from the [Ip2Region Community](https://ip2region.net/products/offline) or third-party vendors. You can try to update the data yourself using the following methods:
|
||||||
|
|
||||||
### 手动编辑更新
|
### Manual Editing and Updating
|
||||||
你可以基于 ip2region 自带的 `./data/ipv4_source.txt` 和 `./data/ipv6_source.txt` 原始 IP 数据用 ip2region 提供的编辑工具来自己修改,目前数据源有如下几种方式:
|
|
||||||
1. ip2region 社区提供的数据(请参考地底部的公众号关注社区通知)
|
|
||||||
2. ip2region Github/Gitee 中带有 `[数据源补充]` 标签的 Issue
|
|
||||||
3. 其他自定义数据:例如客户提供的数据,或者通过 GPS 和 WIFI 定位得到的数据,或者来自其他平台的合法合规的数据
|
|
||||||
|
|
||||||
原始 IP 数据编辑工具使用方法请参考如下的 `maker` 生成程序下的 ReadMe 文档:
|
You can modify the data yourself based on the raw IP data provided by ip2region in `./data/ipv4_source.txt` and `./data/ipv6_source.txt` using the editing tools provided by ip2region. Currently, the data sources include:
|
||||||
| 编程语言 | 描述 | IPv4 支持 | IPv6 支持 | 贡献者 |
|
|
||||||
|:------------------------------------|:-------------------------|:-------------------|:-------------------|:-------------------------------------------|
|
1. Data provided by the ip2region community (please refer to the official account at the bottom for community notifications)
|
||||||
| [Golang](maker/golang#xdb-数据编辑) | golang IP 原始数据编辑器 | :white_check_mark: | :white_check_mark: | [Lion](https://github.com/lionsoul2014) |
|
2. Project Issues tagged with `[Data_Updates]`
|
||||||
| [Java](maker/java#xdb-数据编辑) | java IP 原始数据编辑器 | :white_check_mark: | :x: | [Lion](https://github.com/lionsoul2014) |
|
3. Other custom data: e.g., data provided by customers, data obtained through GPS and WIFI positioning, or legal and compliant data from other platforms.
|
||||||
| [C++](maker/cpp) | C++ IP 原始数据编辑器 | :white_check_mark: | :white_check_mark: | [Yunbin Liu](https://github.com/liuyunbin) |
|
|
||||||
|
For instructions on using the raw IP data editing tools, please refer to the README documents under the following `maker` generation programs:
|
||||||
|
|
||||||
|
| Language | Description | IPv4 Support | IPv6 Support |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| [Golang](maker/golang/README.md#xdb-data-editing) | golang IP raw data editor | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C++](maker/cpp/README.md) | C++ IP raw data editor | :white_check_mark: | :white_check_mark: |
|
||||||
|
|
||||||
|
### Detection Automatic Update
|
||||||
|
|
||||||
|
If you want to update data via your own API or data source, you can refer to the update algorithm based on the "Detection Algorithm" shared in the following videos to write your own update program:
|
||||||
|
|
||||||
|
1. [Data Update Implementation Video Sharing - part1](https://www.bilibili.com/video/BV1934y1E7Q5/)
|
||||||
|
2. [Data Update Implementation Video Sharing - part2](https://www.bilibili.com/video/BV1pF411j7Aw/)
|
||||||
|
|
||||||
|
|
||||||
### 检测自动更新
|
# Official Community
|
||||||
如果你想通过你自己的 API 或数据源来更新数据,你可以参考以下视频分享的 `基于检测算法` 的更新算法来自己编写一个更新程序:
|
|
||||||
1. [数据更新实现视频分享 - part1](https://www.bilibili.com/video/BV1934y1E7Q5/)
|
|
||||||
2. [数据更新实现视频分享 - part2](https://www.bilibili.com/video/BV1pF411j7Aw/)
|
|
||||||
|
|
||||||
# 官方社区
|
The Ip2Region official community was officially launched on `2025/06/12`. On one hand, it provides stable [commercial offline data](https://ip2region.net/products/offline) services. On the other hand, it facilitates the strengthening of the IP toolchain and data services outside the core code, such as [usage documentation](https://ip2region.net/doc/), [query testing](https://ip2region.net/search/demo), and data correction. For more information and services regarding the community, please visit the [Ip2Region Official Community](https://ip2region.net/).
|
||||||
Ip2Region 官方社区正式上线于 `2025/06/12` 日,一方面提供了稳定的 [商用离线数据](https://ip2region.net/products/offline) 服务,另一方面便于在核心代码外强化 IP 工具链和数据服务,例如 [使用文档](https://ip2region.net/doc/),[查询测试](https://ip2region.net/search/demo),数据纠错等,更多关于社区的信息和服务请访问 [Ip2Region 官方社区](https://ip2region.net/)。
|
|
||||||
|
|
||||||
# 相关备注
|
|
||||||
|
|
||||||
### 1、xdb 技术文档:
|
# Related Remarks
|
||||||
1. xdb 数据结构分析:[“ip2region xdb-数据结构描述“](https://ip2region.net/doc/xdb/structure)
|
|
||||||
2. xdb 查询过程分析:[“ip2region xdb-查询过程描述”](https://ip2region.net/doc/xdb/search)
|
|
||||||
3. xdb 生成过程分析:[“ip2region xdb-生成过程描述”](https://ip2region.net/doc/xdb/generate)
|
|
||||||
4. xdb 文件生成教程:[“ip2region xdb-文件生成教程”](https://ip2region.net/doc/data/xdb_make)
|
|
||||||
5. xdb 并发安全查询:[“ip2region xdb-并发安全查询”](https://ip2region.net/doc/xdb/concurrent)
|
|
||||||
6. xdb 数据更新方法:[“ip2region 数据更新和 xdb 数据编辑器的使用”](https://mp.weixin.qq.com/s/cZH5qIn4E5rQFy6N32RCzA)
|
|
||||||
|
|
||||||
### 3、技术信息博客
|
### 1. xdb Technical Documents:
|
||||||
1. 微信公众号 - lionsoul-org,作者活跃的技术分享渠道
|
|
||||||
2. [Ip2Region 官方社区](https://ip2region.net)
|
1. xdb Data Structure Analysis: ["ip2region xdb - Data Structure Description"](https://ip2region.net/doc/xdb/structure)
|
||||||
|
2. xdb Query Process Analysis: ["ip2region xdb - Query Process Description"](https://ip2region.net/doc/xdb/search)
|
||||||
|
3. xdb Generation Process Analysis: ["ip2region xdb - Generation Process Description"](https://ip2region.net/doc/xdb/generate)
|
||||||
|
4. xdb File Generation Tutorial: ["ip2region xdb - File Generation Tutorial"](https://ip2region.net/doc/data/xdb_make)
|
||||||
|
5. xdb Concurrent Safety Query: ["ip2region xdb - Concurrent Safety Query"](https://ip2region.net/doc/xdb/concurrent)
|
||||||
|
6. xdb Data Update Method: ["ip2region Data Update and Use of xdb Data Editor"](https://mp.weixin.qq.com/s/cZH5qIn4E5rQFy6N32RCzA)
|
||||||
|
|
||||||
|
### 3. Technical Information Blogs
|
||||||
|
|
||||||
|
1. WeChat Official Account - lionsoul-org, the author's active technical sharing channel
|
||||||
|
2. [Ip2Region Official Community](https://ip2region.net)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,117 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region
|
||||||
|
|
||||||
|
[ip2region](https://ip2region.net) - 是一个离线IP地址定位库和IP定位数据管理框架,同时支持 `IPv4` 和 `IPv6` ,10微秒级别的查询效率,提供了众多主流编程语言的 `xdb` 数据生成和查询客户端实现。
|
||||||
|
|
||||||
|
|
||||||
|
# 项目特性
|
||||||
|
|
||||||
|
### 1、离线定位库
|
||||||
|
|
||||||
|
项目本身同时了提供了一份 IPv4 (`data/ipv4_source.txt`) 和 IPv6 (`data/ipv6_source.txt`) 的原始数据和对应的 xdb 文件(`data/ip2region_v4.xdb` 和 `data/ip2region_v6.xdb`) 用于实现精确到城市的查询定位功能,字段格式为:`国家|省份|城市|ISP|iso-alpha2-code(国家两字母简称)`,中国的定位信息全部为中文,非中国地区的地域信息全部为英文。
|
||||||
|
|
||||||
|
### 2、数据管理框架
|
||||||
|
|
||||||
|
`xdb` 支持亿级别的 IP 数据段行数,region 信息支持完全自定义,自带数据的 region 信息固定了格式为:`国家|省份|城市|ISP|iso-alpha2-Code`,你可以在 region 中追加特定业务需求的数据,例如:GPS信息/国际统一地域信息编码/邮编等。也就是你完全可以使用 ip2region 来管理你自己的 IP 定位数据。
|
||||||
|
|
||||||
|
### 3、数据去重和压缩
|
||||||
|
|
||||||
|
`xdb` 格式生成程序会自动处理输入的原始数据,检查并且完成相连 IP 段的的合并以及相同地域信息的去重和压缩。
|
||||||
|
|
||||||
|
### 4、极速查询响应
|
||||||
|
|
||||||
|
即使是完全基于 `xdb` 文件的查询,单次查询响应时间在十微秒级别,可通过如下两种方式开启内存加速查询:
|
||||||
|
|
||||||
|
1. `vIndex` 索引缓存 :使用固定的 `512KiB` 的内存空间缓存 vector index 数据,减少一次 IO 磁盘操作,保持平均查询效率稳定在100微秒之内。
|
||||||
|
2. `xdb` 整个文件缓存:将整个 `xdb` 文件全部加载到内存,内存占用等同于 `xdb` 文件大小,无磁盘 IO 操作,保持10微秒级别的查询效率。
|
||||||
|
|
||||||
|
### 5、统一的查询接口
|
||||||
|
|
||||||
|
`xdb` 提供了版本兼容的查询实现,一个统一的 API 可以同时提供对 IPv4 和 IPv6 数据的查询并且返回统一的数据。
|
||||||
|
|
||||||
|
|
||||||
|
# `xdb` 查询
|
||||||
|
|
||||||
|
API 介绍,使用文档和测试程序请参考对应 `searcher` 查询客户端下的 README 介绍,全部查询 binding 实现情况如下:
|
||||||
|
| 编程语言 | 描述 | IPv4 支持 | IPv6 支持 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| [Golang](binding/golang/README_zh.md) | golang 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [PHP](binding/php/README_zh.md) | php 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Java](binding/java/README_zh.md) | java 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C](binding/c/README_zh.md) | C[std=c99] 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Lua_c](binding/lua_c/README_zh.md) | lua c 扩展查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Lua](binding/lua/README_zh.md) | lua 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Rust](binding/rust/README_zh.md) | rust 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Python](binding/python/README_zh.md) | python 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Javascript](binding/javascript/README_zh.md) | javascript 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Csharp](binding/csharp) | csharp 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Erlang](binding/erlang/README_zh.md) | erlang 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Nginx](binding/nginx) | nginx 扩展查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C++](binding/cpp/README_zh.md) | C++ 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Cangjie](binding/cangjie/README.md) | 仓颉 查询客户端 | :white_check_mark: | :white_check_mark: |
|
||||||
|
|
||||||
|
|
||||||
|
以下工具链实现由社区开发者通过第三方仓库贡献:
|
||||||
|
| 编程语言 | 描述 |
|
||||||
|
| --- | --- |
|
||||||
|
| [ip2region-composer](https://github.com/zoujingli/ip2region) | php composer 管理客户端 |
|
||||||
|
| [ip2region-ts](https://github.com/Steven-Qiang/ts-ip2region2) | node.js addon 管理客户端|
|
||||||
|
| [ruby-ip2region](https://github.com/jicheng1014/ruby-ip2region) | ruby xdb 查询客户端实现 |
|
||||||
|
| [Ip2regionTool](https://github.com/orestonce/Ip2regionTool) | ip2region 数据转换工具 |
|
||||||
|
|
||||||
|
|
||||||
|
# `xdb` 生成
|
||||||
|
|
||||||
|
API 介绍,使用文档和测试程序请参考如下 `maker` 生成程序下的 README 文档:
|
||||||
|
|
||||||
|
| 编程语言 | 描述 | IPv4 支持 | IPv6 支持 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| [Golang](maker/golang/README_zh.md) | golang xdb 生成程序 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Java](maker/java/README_zh.md) | java xdb 生成程序 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [Python](maker/python/README_zh.md) | python xdb 生成程序 | :white_check_mark: | :x: |
|
||||||
|
| [Csharp](maker/csharp/README_zh.md) | csharp xdb 生成程序 | :white_check_mark: | :x: |
|
||||||
|
| [Rust](maker/rust/README_zh.md) | rust xdb 生成程序 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C++](maker/cpp) | C++ xdb 生成程序 | :white_check_mark: | :white_check_mark: |
|
||||||
|
|
||||||
|
|
||||||
|
# `xdb` 更新
|
||||||
|
|
||||||
|
ip2region 项目的核心在于 <b>研究 IP 数据的存储和快速查询的设计和实现</b>, 项目自带的 `./data/ipv4_source.txt` 和 `./data/ipv6_source.txt` 原始数据不定期更新,对于数据精度和更新频率要求很高的使用场景建议到 [Ip2Region社区](https://ip2region.net/products/offline) 或者第三方购买商用离线数据,你可以使用如下几种方式来尝试自己更新数据:
|
||||||
|
|
||||||
|
### 手动编辑更新
|
||||||
|
你可以基于 ip2region 自带的 `./data/ipv4_source.txt` 和 `./data/ipv6_source.txt` 原始 IP 数据用 ip2region 提供的编辑工具来自己修改,目前数据源有如下几种方式:
|
||||||
|
1. ip2region 社区提供的数据(请参考地底部的公众号关注社区通知)
|
||||||
|
2. ip2region Github/Gitee 中带有 `[数据源补充]` 标签的 Issue
|
||||||
|
3. 其他自定义数据:例如客户提供的数据,或者通过 GPS 和 WIFI 定位得到的数据,或者来自其他平台的合法合规的数据
|
||||||
|
|
||||||
|
原始 IP 数据编辑工具使用方法请参考如下的 `maker` 生成程序下的 README 文档:
|
||||||
|
| 编程语言 | 描述 | IPv4 支持 | IPv6 支持 |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| [Golang](maker/golang/README_zh.md#xdb-数据编辑) | golang IP 原始数据编辑器 | :white_check_mark: | :white_check_mark: |
|
||||||
|
| [C++](maker/cpp) | C++ IP 原始数据编辑器 | :white_check_mark: | :white_check_mark: |
|
||||||
|
|
||||||
|
|
||||||
|
### 检测自动更新
|
||||||
|
如果你想通过你自己的 API 或数据源来更新数据,你可以参考以下视频分享的 `基于检测算法` 的更新算法来自己编写一个更新程序:
|
||||||
|
1. [数据更新实现视频分享 - part1](https://www.bilibili.com/video/BV1934y1E7Q5/)
|
||||||
|
2. [数据更新实现视频分享 - part2](https://www.bilibili.com/video/BV1pF411j7Aw/)
|
||||||
|
|
||||||
|
|
||||||
|
# 官方社区
|
||||||
|
Ip2Region 官方社区正式上线于 `2025/06/12` 日,一方面提供了稳定的 [商用离线数据](https://ip2region.net/products/offline) 服务,另一方面便于在核心代码外强化 IP 工具链和数据服务,例如 [使用文档](https://ip2region.net/doc/),[查询测试](https://ip2region.net/search/demo),数据纠错等,更多关于社区的信息和服务请访问 [Ip2Region 官方社区](https://ip2region.net/)。
|
||||||
|
|
||||||
|
|
||||||
|
# 相关备注
|
||||||
|
|
||||||
|
### 1、xdb 技术文档:
|
||||||
|
1. xdb 数据结构分析:[“ip2region xdb-数据结构描述“](https://ip2region.net/doc/xdb/structure)
|
||||||
|
2. xdb 查询过程分析:[“ip2region xdb-查询过程描述”](https://ip2region.net/doc/xdb/search)
|
||||||
|
3. xdb 生成过程分析:[“ip2region xdb-生成过程描述”](https://ip2region.net/doc/xdb/generate)
|
||||||
|
4. xdb 文件生成教程:[“ip2region xdb-文件生成教程”](https://ip2region.net/doc/data/xdb_make)
|
||||||
|
5. xdb 并发安全查询:[“ip2region xdb-并发安全查询”](https://ip2region.net/doc/xdb/concurrent)
|
||||||
|
6. xdb 数据更新方法:[“ip2region 数据更新和 xdb 数据编辑器的使用”](https://mp.weixin.qq.com/s/cZH5qIn4E5rQFy6N32RCzA)
|
||||||
|
|
||||||
|
### 3、技术信息博客
|
||||||
|
1. 微信公众号 - lionsoul-org,作者活跃的技术分享渠道
|
||||||
|
2. [Ip2Region 官方社区](https://ip2region.net)
|
||||||
|
|
@ -15,7 +15,7 @@ xdb_util.o: xdb_util.c
|
||||||
xdb_searcher_lib: xdb_util.o xdb_searcher.o
|
xdb_searcher_lib: xdb_util.o xdb_searcher.o
|
||||||
mkdir -p build/lib
|
mkdir -p build/lib
|
||||||
mkdir -p build/include
|
mkdir -p build/include
|
||||||
ar -rc build/lib/libxdb.a `find . -name *.o`
|
ar -rc build/lib/libxdb.a `find . -name "*.o"`
|
||||||
cp xdb_api.h build/include
|
cp xdb_api.h build/include
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,370 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region c Query Client
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### About Query API
|
||||||
|
|
||||||
|
The prototype of the Query API is as follows:
|
||||||
|
|
||||||
|
```c
|
||||||
|
// Query via string IP
|
||||||
|
int xdb_search_by_string(xdb_searcher_t *, const string_ip_t *, xdb_region_buffer_t *);
|
||||||
|
// Query via binary IP returned by xdb_parse_ip
|
||||||
|
int xdb_search(xdb_searcher_t *, const bytes_ip_t *, int, xdb_region_buffer_t *);
|
||||||
|
```
|
||||||
|
|
||||||
|
If the query fails, a non-`0` error code will be returned. If the query is successful, the `region` information string can be obtained from `xdb_region_buffer_t`. If the input IP cannot be found, `xdb_region_buffer_t` will receive an empty string `""`.
|
||||||
|
|
||||||
|
### About IPv4 and IPv6
|
||||||
|
|
||||||
|
This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows:
|
||||||
|
|
||||||
|
```c
|
||||||
|
#include "xdb_api.h";
|
||||||
|
|
||||||
|
// For IPv4: Set xdb path to the v4 xdb file, specify IP version as IPv4
|
||||||
|
const char *db_path = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path
|
||||||
|
xdb_version_t *version = XDB_IPv4;
|
||||||
|
|
||||||
|
// For IPv6: Set xdb path to the v6 xdb file, specify IP version as IPv6
|
||||||
|
const char *db_path = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path
|
||||||
|
xdb_version_t *version = XDB_IPv6;
|
||||||
|
|
||||||
|
// The IP version of the xdb specified by db_path must be consistent with the version, otherwise an error will occur during query execution
|
||||||
|
// Note: The following demonstration directly uses db_path and version variables
|
||||||
|
```
|
||||||
|
|
||||||
|
### XDB File Verification
|
||||||
|
|
||||||
|
It is recommended that you proactively verify the applicability of the xdb file, as some future new features may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify when the service starts or manually call a command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
|
```c
|
||||||
|
#include "xdb_api.h";
|
||||||
|
|
||||||
|
int errcode = xdb_verify(db_path);
|
||||||
|
if ($err != 0) {
|
||||||
|
// Applicability verification failed!!!
|
||||||
|
// The current query client implementation is not suitable for querying the xdb file specified by db_path.
|
||||||
|
// You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with db_path.
|
||||||
|
printf("failed to verify xdb file `%s`, errcode: %d\n", db_path, errcode);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by dbPath
|
||||||
|
```
|
||||||
|
|
||||||
|
### File-Based Query
|
||||||
|
|
||||||
|
```c
|
||||||
|
#include <stdio.h>
|
||||||
|
#include "xdb_api.h"
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
xdb_searcher_t searcher;
|
||||||
|
char region_buffer[512] = {'\0'};
|
||||||
|
xdb_region_buffer_t region;
|
||||||
|
|
||||||
|
// Initialize region_buffer_t using region_buffer from stack space
|
||||||
|
int err = xdb_region_buffer_init(®ion, region_buffer, sizeof(region_buffer));
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to init the region buffer with errcode=%d\n", err);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize winsock when the service starts; no need to call repeatedly, only needed on Windows systems
|
||||||
|
err = xdb_init_winsock();
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to init the winsock with errno=%d\n", err);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Initialize xdb query object from db_path.
|
||||||
|
// @Note: Use the db_path and version described above to create the searcher
|
||||||
|
err = xdb_new_with_file_only(version, &searcher, db_path);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to create xdb searcher from `%s` with errno=%d\n", db_path, err);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Call search API to query, both IPv4 and IPv6 are supported.
|
||||||
|
const char *ip_string = "1.2.3.4";
|
||||||
|
// ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
|
||||||
|
long cost_time = 0, s_time = xdb_now();
|
||||||
|
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
||||||
|
cost_time = (int) (xdb_now() - s_time);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed search(%s) with errno=%d\n", ip_string, err);
|
||||||
|
} else {
|
||||||
|
printf("{region: %s, took: %d μs}", region.value, cost_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up memory resources for region info; must be called after every search
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
|
||||||
|
// Note: For concurrent use, each thread needs to define and initialize its own searcher query object independently.
|
||||||
|
|
||||||
|
// 3. Close xdb searcher
|
||||||
|
xdb_close(&searcher);
|
||||||
|
xdb_clean_winsock(); // Call on Windows
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
We can pre-load VectorIndex data from the xdb file and cache it globally. Using the global VectorIndex cache every time a Searcher object is created can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure.
|
||||||
|
|
||||||
|
```c
|
||||||
|
#include <stdio.h>
|
||||||
|
#include "xdb_api.h"
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
xdb_vector_index_t *v_index;
|
||||||
|
xdb_searcher_t searcher;
|
||||||
|
xdb_region_buffer_t region;
|
||||||
|
|
||||||
|
// Initialize region_buffer with NULL to let it manage memory allocation automatically
|
||||||
|
int err = xdb_region_buffer_init(®ion, NULL, 0);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to init the region buffer with errcode=%d\n", err);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize winsock when the service starts; no need to call repeatedly, only needed on Windows systems
|
||||||
|
err = xdb_init_winsock();
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to init the winsock with errno=%d\n", err);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Load VectorIndex from the db_path described above.
|
||||||
|
// Obtain v_index to create a global cache for subsequent repeated use.
|
||||||
|
// Note: v_index does not need to be loaded every time; it is recommended to load it once at service startup as a global resource.
|
||||||
|
v_index = xdb_load_vector_index_from_file(db_path);
|
||||||
|
if (v_index == NULL) {
|
||||||
|
printf("failed to load vector index from `%s`\n", db_path);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Use the global VectorIndex variable to create an xdb searcher with VectorIndex cache.
|
||||||
|
// @Note: Use the db_path and version described above to create the searcher
|
||||||
|
err = xdb_new_with_vector_index(version, &searcher, db_path, v_index);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to create vector index cached searcher with errcode=%d\n", err);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// 3. Call search API to query, both IPv4 and IPv6 are supported
|
||||||
|
const char *ip_string = "1.2.3.4";
|
||||||
|
// ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
|
||||||
|
long cost_time = 0, s_time = xdb_now();
|
||||||
|
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
||||||
|
cost_time = (int) (xdb_now() - s_time);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed search(%s) with errno=%d\n", ip_string, err);
|
||||||
|
} else {
|
||||||
|
printf("{region: %s, took: %d μs}", region.value, cost_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up memory resources for region info; must be called after every search
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
|
||||||
|
|
||||||
|
// Note: For concurrent use, each thread needs to define and initialize its own searcher query object independently.
|
||||||
|
|
||||||
|
// 4. Close xdb searcher; if the service is being shut down, the memory for v_index also needs to be freed.
|
||||||
|
xdb_close(&searcher);
|
||||||
|
xdb_close_vector_index(v_index);
|
||||||
|
xdb_clean_winsock();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching the Entire `xdb` File
|
||||||
|
|
||||||
|
We can also pre-load the entire xdb file into memory and then create a query object based on this data to achieve fully memory-based queries, similar to the previous memory search.
|
||||||
|
|
||||||
|
```c
|
||||||
|
#include <stdio.h>
|
||||||
|
#include "xdb_api.h"
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
xdb_content_t *c_buffer;
|
||||||
|
xdb_searcher_t searcher;
|
||||||
|
xdb_region_buffer_t region;
|
||||||
|
|
||||||
|
// Initialize region_buffer with NULL to let it manage memory allocation automatically
|
||||||
|
int err = xdb_region_buffer_init(®ion, NULL, 0);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to init the region buffer with errcode=%d\n", err);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Initialize winsock when the service starts; no need to call repeatedly, only needed on Windows systems
|
||||||
|
err = xdb_init_winsock();
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to init the winsock with errno=%d\n", err);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Load the entire xdb data from the db_path described above.
|
||||||
|
c_buffer = xdb_load_content_from_file(db_path);
|
||||||
|
if (v_index == NULL) {
|
||||||
|
printf("failed to load xdb content from `%s`\n", db_path);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Use the global c_buffer variable to create a fully memory-based xdb query object.
|
||||||
|
// @Note: Use the version described above to create the searcher.
|
||||||
|
err = xdb_new_with_buffer(version, &searcher, c_buffer);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed to create content cached searcher with errcode=%d\n", err);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Call search API to query, both IPv4 and IPv6 are supported
|
||||||
|
const char *ip_string = "1.2.3.4";
|
||||||
|
// ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
|
||||||
|
long cost_time = 0, s_time = xdb_now();
|
||||||
|
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
||||||
|
cost_time = (int) (xdb_now() - s_time);
|
||||||
|
if (err != 0) {
|
||||||
|
printf("failed search(%s) with errno=%d\n", ip_string, err);
|
||||||
|
} else {
|
||||||
|
printf("{region: %s, took: %d μs}", region.value, cost_time);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up memory resources for region info; must be called after every search
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
|
||||||
|
|
||||||
|
// Note: For concurrent use, xdb query objects created this way can be safely used for concurrency.
|
||||||
|
// It is recommended to create them when the service starts and then use them safely in parallel until the service shuts down.
|
||||||
|
|
||||||
|
// 4. Close xdb searcher; memory for c_buffer needs to be freed when shutting down the service.
|
||||||
|
xdb_close(&searcher);
|
||||||
|
xdb_close_content(c_buffer);
|
||||||
|
xdb_clean_winsock();
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### About Storage of Location Information
|
||||||
|
|
||||||
|
In older implementations, search-related functions relied on a specified `region_buffer` memory to store location information, which had significant limitations.
|
||||||
|
The new implementation provides an `xdb_region_buffer_t` object to manage these memory allocations. You can still specify a fixed `region_buffer` to create memory management for the region; this is suitable when the maximum length of your location information is known, which helps reduce memory fragmentation during runtime. If the length of the location information is uncertain or if your program is not suited for pre-allocating a block of memory, you can initialize `xdb_region_buffer_t` by specifying `NULL`. In this case, the object will automatically manage memory allocation, making it suitable for storing location information of any length, though this approach will certainly increase memory fragmentation over long-term operation.
|
||||||
|
|
||||||
|
```c
|
||||||
|
// 1. Create region_buffer by specifying a memory block
|
||||||
|
char buffer[512];
|
||||||
|
xdb_region_buffer_t region;
|
||||||
|
int err = xdb_region_buffer_init(®ion, buffer, sizeof(buffer));
|
||||||
|
if (err != 0) {
|
||||||
|
// Initialization failed
|
||||||
|
printf("failed to init region buffer width errcode=%d", err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Create region_buffer by specifying NULL to let it allocate memory as needed automatically
|
||||||
|
xdb_region_buffer_t region;
|
||||||
|
int err = xdb_region_buffer_init(®ion, NULL, 0);
|
||||||
|
if (err != 0) {
|
||||||
|
// Initialization failed
|
||||||
|
printf("failed to init region buffer width errcode=%d", err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Note: After each query call, you must manually call the function to free memory.
|
||||||
|
// The search function will report an error if used with uncleaned region info.
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
```
|
||||||
|
|
||||||
|
# Compiling the Test Program
|
||||||
|
|
||||||
|
Compile and obtain the `xdb_searcher` executable as follows:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# cd to the c binding root directory
|
||||||
|
➜ c git:(master) ✗ make
|
||||||
|
gcc -std=c99 -Wall -O2 -I./ xdb_util.c xdb_searcher.c main.c -o xdb_searcher
|
||||||
|
gcc -std=c99 -Wall -O2 -I./ xdb_util.c test_util.c -o test_util
|
||||||
|
```
|
||||||
|
|
||||||
|
# Query Testing
|
||||||
|
|
||||||
|
Test queries against xdb via the `xdb_searcher search` command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search
|
||||||
|
./xdb_searcher search [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: performing IPv4 query testing using the default data/ip2region_v4.xdb:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search --db=../../data/ip2region_v4.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 39 μs}
|
||||||
|
ip2region>> 120.229.45.2
|
||||||
|
{region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 13 μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: performing IPv6 query testing using the default data/ip2region_v6.xdb:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher search --db=../../data/ip2region_v6.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> ::
|
||||||
|
{region: , io_count: 1, took: 38 μs}
|
||||||
|
ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff
|
||||||
|
{region: United States|Florida|Miami|velia.net Internetdienste GmbH|US, io_count: 14, took: 76 μs}
|
||||||
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 42 μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
Enter an IP to perform a query; enter `quit` to exit the test program. You can also set `cache-policy` to file/vectorIndex/content respectively to test the efficiency of the three different cache implementations.
|
||||||
|
|
||||||
|
# bench Testing
|
||||||
|
|
||||||
|
Perform bench testing via the `xdb_searcher bench` command. This ensures there are no errors in the query program and the `xdb` file, while also providing average query performance through a large number of queries:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench
|
||||||
|
./xdb_searcher bench [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--src string source ip text file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: performing IPv4 bench testing via the default data/ip2region_v4.xdb and data/ipv4_source.txt:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
Bench finished, {cache_policy: vectorIndex, total: 1367686, took: 7.640s, cost: 5 μs/op}
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: performing IPv6 bench testing via the default data/ip2region_v6.xdb and data/ipv6_source.txt:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ c git:(fr_c_ipv6) ✗ ./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
Bench finished, {cache_policy: vectorIndex, total: 34159862, took: 857.750s, cost: 24 μs/op}
|
||||||
|
```
|
||||||
|
|
||||||
|
You can set the `cache-policy` parameter to test the efficiency of different cache mechanisms (file/vectorIndex/content). @Note: Please ensure that the `src` file used for benching is the same source file used to generate the corresponding `xdb` file.
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# ip2region xdb c 查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region c 查询客户端
|
||||||
|
|
||||||
|
|
||||||
# 使用方式
|
# 使用方式
|
||||||
|
|
@ -82,7 +84,7 @@ int main(int argc, char *argv[]) {
|
||||||
|
|
||||||
// 2、调用 search API 查询,IPv4 和 IPv6 都支持.
|
// 2、调用 search API 查询,IPv4 和 IPv6 都支持.
|
||||||
const char *ip_string = "1.2.3.4";
|
const char *ip_string = "1.2.3.4";
|
||||||
// ip_string = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
|
||||||
long cost_time = 0, s_time = xdb_now();
|
long cost_time = 0, s_time = xdb_now();
|
||||||
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
||||||
|
|
@ -151,7 +153,7 @@ int main(int argc, char *argv[]) {
|
||||||
|
|
||||||
// 3、调用 search API 查询,IPv4 和 IPv6 都支持
|
// 3、调用 search API 查询,IPv4 和 IPv6 都支持
|
||||||
const char *ip_string = "1.2.3.4";
|
const char *ip_string = "1.2.3.4";
|
||||||
// ip_string = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
|
||||||
long cost_time = 0, s_time = xdb_now();
|
long cost_time = 0, s_time = xdb_now();
|
||||||
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
||||||
|
|
@ -176,7 +178,7 @@ int main(int argc, char *argv[]) {
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
我们也可以预先加载整个 xdb 文件到内存,然后基于这个数据创建查询对象来实现完全基于内存的查询,类似之前的 memory search。
|
我们也可以预先加载整个 xdb 文件到内存,然后基于这个数据创建查询对象来实现完全基于内存的查询,类似之前的 memory search。
|
||||||
```c
|
```c
|
||||||
|
|
@ -220,7 +222,7 @@ int main(int argc, char *argv[]) {
|
||||||
|
|
||||||
// 3、调用 search API 查询,IPv4 和 IPv6 都支持
|
// 3、调用 search API 查询,IPv4 和 IPv6 都支持
|
||||||
const char *ip_string = "1.2.3.4";
|
const char *ip_string = "1.2.3.4";
|
||||||
// ip_string = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip_string = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
|
||||||
long cost_time = 0, s_time = xdb_now();
|
long cost_time = 0, s_time = xdb_now();
|
||||||
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
err = xdb_search_by_string(&searcher, ip_string, ®ion);
|
||||||
|
|
@ -304,8 +306,10 @@ options:
|
||||||
ip2region xdb searcher test program
|
ip2region xdb searcher test program
|
||||||
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 39 μs}
|
||||||
ip2region>> 120.229.45.2
|
ip2region>> 120.229.45.2
|
||||||
{region: 中国|广东省|深圳市|移动, io_count: 3, took: 29 μs}
|
{region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 13 μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
例如:使用默认的 data/ip2region_v6.xdb 进行 IPv6 查询测试:
|
例如:使用默认的 data/ip2region_v6.xdb 进行 IPv6 查询测试:
|
||||||
|
|
@ -317,9 +321,9 @@ type 'quit' to exit
|
||||||
ip2region>> ::
|
ip2region>> ::
|
||||||
{region: , io_count: 1, took: 38 μs}
|
{region: , io_count: 1, took: 38 μs}
|
||||||
ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff
|
ip2region>> 2604:bc80:8001:11a4:ffff:ffff:ffff:ffff
|
||||||
{region: 美国|特拉华州|刘易斯|数据中心, io_count: 14, took: 77 μs}
|
{region: United States|Florida|Miami|velia.net Internetdienste GmbH|US, io_count: 14, took: 76 μs}
|
||||||
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, io_count: 8, took: 46 μs}
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 42 μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
输入 ip 即可进行查询,输入 quit 即可退出测试程序。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同的缓存实现的效率。
|
输入 ip 即可进行查询,输入 quit 即可退出测试程序。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同的缓存实现的效率。
|
||||||
|
|
@ -9,6 +9,10 @@
|
||||||
#ifndef C_IP2REGION_XDB_H
|
#ifndef C_IP2REGION_XDB_H
|
||||||
#define C_IP2REGION_XDB_H
|
#define C_IP2REGION_XDB_H
|
||||||
|
|
||||||
|
// @Note:
|
||||||
|
// this define must be put before any header include
|
||||||
|
// force the LFS for ftell
|
||||||
|
#define _FILE_OFFSET_BITS 64
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
@ -201,6 +205,11 @@ XDB_PUBLIC(int) xdb_v6_ip_to_string(const bytes_ip_t *, char *, size_t);
|
||||||
// returns: -1 if ip1 < ip2, 1 if ip1 > ip2 or 0
|
// returns: -1 if ip1 < ip2, 1 if ip1 > ip2 or 0
|
||||||
XDB_PUBLIC(int) xdb_ip_sub_compare(const bytes_ip_t *, int, const char *, int);
|
XDB_PUBLIC(int) xdb_ip_sub_compare(const bytes_ip_t *, int, const char *, int);
|
||||||
|
|
||||||
|
// large file seek and tell
|
||||||
|
XDB_PUBLIC(int) xdb_fseek(FILE *, long long, int);
|
||||||
|
|
||||||
|
XDB_PUBLIC(long long) xdb_ftell(FILE *);
|
||||||
|
|
||||||
// --- END xdb utils
|
// --- END xdb utils
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -196,6 +196,13 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, const bytes_ip_t *ip_bytes, int
|
||||||
}
|
}
|
||||||
|
|
||||||
// printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr);
|
// printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr);
|
||||||
|
// @Note: ptr validate, zero ptr means source data missing
|
||||||
|
// so we could just stop here and return an empty string.
|
||||||
|
if (s_ptr == 0 || e_ptr == 0) {
|
||||||
|
xdb_region_buffer_empty(region);
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
// binary search to get the final region info
|
// binary search to get the final region info
|
||||||
// segment_buffer = xdb_malloc(seg_index_size);
|
// segment_buffer = xdb_malloc(seg_index_size);
|
||||||
seg_index_size = xdb->version->segment_index_size;
|
seg_index_size = xdb->version->segment_index_size;
|
||||||
|
|
|
||||||
|
|
@ -228,7 +228,7 @@ XDB_PUBLIC(void) xdb_free_content(void *ptr) {
|
||||||
}
|
}
|
||||||
|
|
||||||
XDB_PUBLIC(int) xdb_verify_from_header(FILE *handle, xdb_header_t *header) {
|
XDB_PUBLIC(int) xdb_verify_from_header(FILE *handle, xdb_header_t *header) {
|
||||||
int runtime_ptr_bytes = 0; // runtime ptr bytes
|
unsigned int runtime_ptr_bytes = 0; // runtime ptr bytes
|
||||||
if (header->version == xdb_structure_20) {
|
if (header->version == xdb_structure_20) {
|
||||||
runtime_ptr_bytes = 4;
|
runtime_ptr_bytes = 4;
|
||||||
} else if (header->version == xdb_structure_30) {
|
} else if (header->version == xdb_structure_30) {
|
||||||
|
|
@ -244,9 +244,9 @@ XDB_PUBLIC(int) xdb_verify_from_header(FILE *handle, xdb_header_t *header) {
|
||||||
return 3;
|
return 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
long int fileBytes = ftell(handle);
|
long long fileBytes = xdb_ftell(handle);
|
||||||
long int maxFilePtr = (1L << (runtime_ptr_bytes * 8)) - 1;
|
long long maxFilePtr = (1LL << (runtime_ptr_bytes * 8)) - 1;
|
||||||
// printf("fileBytes: %ld, maxFilePtr: %ld\n", fileBytes, maxFilePtr);
|
// printf("fileBytes: %lld, maxFilePtr: %lld\n", fileBytes, maxFilePtr);
|
||||||
if (fileBytes > maxFilePtr) {
|
if (fileBytes > maxFilePtr) {
|
||||||
return 4;
|
return 4;
|
||||||
}
|
}
|
||||||
|
|
@ -496,3 +496,30 @@ XDB_PUBLIC(int) xdb_ip_sub_compare(const bytes_ip_t *ip1, int bytes, const char
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
XDB_PUBLIC(int) xdb_fseek(FILE *handle, long long offset, int whence) {
|
||||||
|
// we may have to use the large file solution later
|
||||||
|
// #if defined(XDB_LINUX)
|
||||||
|
// return fseeko(handle, (off_t) offset, whence);
|
||||||
|
// #elif defined(XDB_WINDOWS)
|
||||||
|
// return _fseeki64(handle, (__int64) offset, whence)
|
||||||
|
// #else
|
||||||
|
// return fseek(handle, (long) offset, whence);
|
||||||
|
// #endif
|
||||||
|
|
||||||
|
return fseek(handle, (long) offset, whence);
|
||||||
|
}
|
||||||
|
|
||||||
|
XDB_PUBLIC(long long) xdb_ftell(FILE *handle) {
|
||||||
|
// we may have to use the large file solution later
|
||||||
|
// #if defined(XDB_LINUX)
|
||||||
|
// return (long long) ftello(handle);
|
||||||
|
// #elif defined(XDB_WINDOWS)
|
||||||
|
// return (long long) _ftelli64(handle);
|
||||||
|
// #else
|
||||||
|
// // report error ?
|
||||||
|
// return (long long) ftell(handle);
|
||||||
|
// #endif
|
||||||
|
|
||||||
|
return (long long) ftell(handle);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
# build cache
|
||||||
|
.cache/
|
||||||
|
|
||||||
|
# dependency lock
|
||||||
|
cjpm.lock
|
||||||
|
|
@ -0,0 +1,172 @@
|
||||||
|
# ip2region Cangjie Query Client
|
||||||
|
|
||||||
|
This implementation supports both `IPv4` and `IPv6`, and provides a `thread-safe` service layer.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
binding/cangjie/
|
||||||
|
├── src/
|
||||||
|
│ ├── xdb/ # Low-level xdb query engine
|
||||||
|
│ │ ├── searcher.cj # Searcher (file-only / vector-index / content-buff modes)
|
||||||
|
│ │ ├── util.cj # IP parsing/comparison, byte readers
|
||||||
|
│ │ ├── header.cj # xdb header + version parsing
|
||||||
|
│ │ ├── version.cj # IPv4/IPv6 version definitions
|
||||||
|
│ │ ├── *_test.cj # Unit tests (24 cases)
|
||||||
|
│ ├── service/ # High-level thread-safe service layer
|
||||||
|
│ │ ├── config.cj # Config (cache policy, pool size, pre-loaded data)
|
||||||
|
│ │ ├── searcher_pool.cj # SearcherPool (Semaphore + ConcurrentLinkedQueue)
|
||||||
|
│ │ ├── ip2region.cj # Ip2Region unified API
|
||||||
|
│ │ ├── *_test.cj # Unit tests (17 cases)
|
||||||
|
├── example/ # Executable CLI demo (separate project)
|
||||||
|
│ ├── cjpm.toml
|
||||||
|
│ └── src/main.cj # 5 usage demos + benchmarks
|
||||||
|
├── cjpm.toml # Static library project
|
||||||
|
└── README.md
|
||||||
|
```
|
||||||
|
|
||||||
|
**Two API layers:**
|
||||||
|
|
||||||
|
- **`ip2region.xdb`** — Low-level Searcher. Not thread-safe (mutable `ioCount` + `File` seek/read state). Supports three cache modes.
|
||||||
|
- **`ip2region.service`** — High-level `Ip2Region` class wrapping `SearcherPool`. Thread-safe, handles v4/v6 dispatch automatically.
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd binding/cangjie
|
||||||
|
cjpm build # static library
|
||||||
|
cd example
|
||||||
|
cjpm build # example executable
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test
|
||||||
|
|
||||||
|
41 tests across both packages (24 xdb + 17 service):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd binding/cangjie
|
||||||
|
cjpm test
|
||||||
|
```
|
||||||
|
|
||||||
|
## Cache Policies
|
||||||
|
|
||||||
|
| Policy | Memory | Speed | Thread-safe* |
|
||||||
|
|--------|--------|-------|-------------|
|
||||||
|
| `FileOnly` (0) | ~0 MB | ~21 µs/op (v4) | Via SearcherPool |
|
||||||
|
| `VectorIndex` (1) | ~4 MB | ~20 µs/op (v4) | Via SearcherPool |
|
||||||
|
| `ContentBuff` (2) | ~full xdb | ~5 µs/op (v4) | Via SearcherPool |
|
||||||
|
|
||||||
|
\* Low-level `Searcher` is NOT thread-safe. The service layer (`SearcherPool` / `Ip2Region`) provides thread safety by pooling searchers (FileOnly/VectorIndex) or sharing a read-only buffer (ContentBuff).
|
||||||
|
|
||||||
|
## Benchmark Results
|
||||||
|
|
||||||
|
Benchmarked on Windows 11, AMD Ryzen 7, 487k IPv4 records / 638k IPv6 records:
|
||||||
|
|
||||||
|
### IPv4 (`ip2region_v4.xdb`, 487,167 records)
|
||||||
|
|
||||||
|
| Cache Policy | Total | Time | Avg |
|
||||||
|
|-------------|-------|------|-----|
|
||||||
|
| ContentBuff | 487,167 | 3,059 ms | **5 µs/op** |
|
||||||
|
| VectorIndex | 487,167 | 10,413 ms | **20 µs/op** |
|
||||||
|
| FileOnly | 487,167 | 11,271 ms | **21 µs/op** |
|
||||||
|
|
||||||
|
### IPv6 (`ip2region_v6.xdb`, 638,953 records)
|
||||||
|
|
||||||
|
| Cache Policy | Total | Time | Avg |
|
||||||
|
|-------------|-------|------|-----|
|
||||||
|
| ContentBuff | 638,953 | 37,080 ms | **56 µs/op** |
|
||||||
|
| VectorIndex | 638,953 | 38,443 ms | **58 µs/op** |
|
||||||
|
| FileOnly | 638,953 | 19,148 ms | **29 µs/op** |
|
||||||
|
|
||||||
|
Run your own:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd example
|
||||||
|
cjpm run -- bench --db ../../../data/ip2region_v4.xdb --src ../../../data/ipv4_source.txt --cache-policy content
|
||||||
|
cjpm run -- bench --db ../../../data/ip2region_v6.xdb --src ../../../data/ipv6_source.txt --cache-policy file
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Low-level API (`ip2region.xdb`)
|
||||||
|
|
||||||
|
```cangjie
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
// Content buffer mode (fastest)
|
||||||
|
let content = File.readFrom(Path("ip2region_v4.xdb"))
|
||||||
|
let header = newHeaderFromBytes(content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let searcher = Searcher(version, content, "ip2region_v4.xdb")
|
||||||
|
let region = searcher.search(parseIP("220.181.108.183"))
|
||||||
|
searcher.close()
|
||||||
|
|
||||||
|
// Vector index mode
|
||||||
|
let vIndex = loadVectorIndex(content)
|
||||||
|
let searcher2 = Searcher(version, "ip2region_v4.xdb", vIndex)
|
||||||
|
searcher2.close()
|
||||||
|
|
||||||
|
// File only mode
|
||||||
|
let searcher3 = Searcher(version, "ip2region_v4.xdb")
|
||||||
|
searcher3.close()
|
||||||
|
```
|
||||||
|
|
||||||
|
### High-level API (`ip2region.service`)
|
||||||
|
|
||||||
|
```cangjie
|
||||||
|
import ip2region.service.*
|
||||||
|
|
||||||
|
// Via factory (VectorIndex, 20 searchers)
|
||||||
|
let region = newIp2Region("ip2region_v4.xdb", "ip2region_v6.xdb")
|
||||||
|
let result = region.search("220.181.108.183")
|
||||||
|
region.close()
|
||||||
|
|
||||||
|
// Custom config
|
||||||
|
let v4Cfg = Config(ContentBuff, IPv4, "ip2region_v4.xdb", 10)
|
||||||
|
let v6Cfg = Config(VectorIndex, IPv6, "ip2region_v6.xdb", 20)
|
||||||
|
let svc = Ip2Region(v4Cfg, v6Cfg)
|
||||||
|
let result = svc.search("2408:8266:100:1000::")
|
||||||
|
svc.close()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Single-version API
|
||||||
|
|
||||||
|
```cangjie
|
||||||
|
import ip2region.service.*
|
||||||
|
|
||||||
|
// IPv4 only — Ip2Region(config, IPv4)
|
||||||
|
let v4Cfg = Config(ContentBuff, IPv4, "ip2region_v4.xdb", 10)
|
||||||
|
let v4 = Ip2Region(v4Cfg, IPv4)
|
||||||
|
let r4 = v4.search("220.181.108.183")
|
||||||
|
v4.close()
|
||||||
|
|
||||||
|
// IPv6 only — Ip2Region(config, IPv6)
|
||||||
|
let v6Cfg = Config(ContentBuff, IPv6, "ip2region_v6.xdb", 10)
|
||||||
|
let v6 = Ip2Region(v6Cfg, IPv6)
|
||||||
|
let r6 = v6.search("2408:8266:100:1000::")
|
||||||
|
v6.close()
|
||||||
|
|
||||||
|
// Factory functions
|
||||||
|
let v4Only = newIp2RegionV4(v4Cfg)
|
||||||
|
let v6Only = newIp2RegionV6(v6Cfg)
|
||||||
|
v4Only.close(); v6Only.close()
|
||||||
|
```
|
||||||
|
|
||||||
|
### CLI
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd example
|
||||||
|
cjpm run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Thread Safety
|
||||||
|
|
||||||
|
- **`xdb.Searcher`** — NOT thread-safe. Keep one per thread or use a pool.
|
||||||
|
- **`service.SearcherPool`** — Thread-safe. Pre-allocates N searchers, uses `Semaphore` for backpressure and `ConcurrentLinkedQueue` for storage. `borrow()` blocks until a searcher is available.
|
||||||
|
- **`service.Ip2Region`** — Thread-safe. Delegates to pool or shared in-mem searcher based on cache policy. ContentBuff mode shares a single Searcher (Array<Byte> is read-only), FileOnly/VectorIndex use the pool.
|
||||||
|
- **`service.Config`** — Immutable after construction. Safe to share.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- Cangjie SDK 1.1.0+
|
||||||
|
- `cjpm` build tool
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
[package]
|
||||||
|
cjc-version = "1.1.0"
|
||||||
|
name = "ip2region"
|
||||||
|
organization = ""
|
||||||
|
description = "ip2region xdb searcher library for Cangjie"
|
||||||
|
version = "1.0.0"
|
||||||
|
target-dir = ""
|
||||||
|
script-dir = ""
|
||||||
|
output-type = "static"
|
||||||
|
compile-option = "-Woff unused"
|
||||||
|
override-compile-option = ""
|
||||||
|
link-option = ""
|
||||||
|
package-configuration = {}
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
[package]
|
||||||
|
cjc-version = "1.1.0"
|
||||||
|
name = "ip2regionExample"
|
||||||
|
version = "1.0.0"
|
||||||
|
output-type = "executable"
|
||||||
|
compile-option = "-Woff unused"
|
||||||
|
description = "ip2region Cangjie CLI example"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
ip2region = { path = ".." }
|
||||||
|
|
@ -0,0 +1,196 @@
|
||||||
|
package ip2regionExample
|
||||||
|
|
||||||
|
import std.fs.*
|
||||||
|
import std.time.*
|
||||||
|
import std.collection.*
|
||||||
|
import ip2region.xdb.*
|
||||||
|
import ip2region.service.*
|
||||||
|
|
||||||
|
// ip2region Cangjie 使用示例
|
||||||
|
//
|
||||||
|
// 构建: cjpm build
|
||||||
|
// 运行: cjpm run
|
||||||
|
//
|
||||||
|
// 默认从 ../../../data/ 查找 xdb 文件,
|
||||||
|
// 也可通过命令行参数指定路径:
|
||||||
|
// cjpm run -- --v4-db <path> --v6-db <path>
|
||||||
|
|
||||||
|
func printHelp() {
|
||||||
|
println("ip2region Cangjie 使用示例")
|
||||||
|
println("用法: cjpm run -- [options]")
|
||||||
|
println("选项:")
|
||||||
|
println(" --v4-db <path> IPv4 xdb 文件路径 (默认: ../../../data/ip2region_v4.xdb)")
|
||||||
|
println(" --v6-db <path> IPv6 xdb 文件路径 (默认: ../../../data/ip2region_v6.xdb)")
|
||||||
|
println(" --help 显示此帮助")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// 方式一: 使用底层 Searcher API (ip2region.xdb)
|
||||||
|
// 可自行选择缓存策略,单线程使用
|
||||||
|
// ============================================================
|
||||||
|
func demoXdbSearcher(v4Path: String, v6Path: String) {
|
||||||
|
println("\n=== 方式一: 底层 Searcher API ===")
|
||||||
|
|
||||||
|
// 1. 读取 xdb 文件到内存
|
||||||
|
let v4Content = File.readFrom(Path(v4Path))
|
||||||
|
|
||||||
|
// 2. 解析文件头,获取版本信息
|
||||||
|
let v4Header = newHeaderFromBytes(v4Content)
|
||||||
|
let v4Version = versionFromHeader(v4Header)
|
||||||
|
|
||||||
|
// 3. 选择缓存策略创建 Searcher
|
||||||
|
// 支持三种模式: ContentBuff(最快), VectorIndex(均衡), FileOnly(省内存)
|
||||||
|
let vIndex = loadVectorIndex(v4Content)
|
||||||
|
let searcher = Searcher(v4Version, v4Path, vIndex) // VectorIndex 模式
|
||||||
|
|
||||||
|
// 4. 查询 IP
|
||||||
|
let ip = parseIP("220.181.108.183")
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
println(" IP: 220.181.108.183")
|
||||||
|
println(" 结果: ${region}")
|
||||||
|
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// 方式二: 使用 Ip2Region 服务 API (ip2region.service)
|
||||||
|
// 自动管理连接池,支持多线程安全访问
|
||||||
|
// ============================================================
|
||||||
|
func demoIp2Region(v4Path: String, v6Path: String) {
|
||||||
|
println("\n=== 方式二: Ip2Region 服务 API ===")
|
||||||
|
|
||||||
|
// 1. 创建配置 (缓存策略 + 连接池大小)
|
||||||
|
let v4Cfg = Config(VectorIndex, IPv4, v4Path, 10)
|
||||||
|
let v6Cfg = Config(VectorIndex, IPv6, v6Path, 10)
|
||||||
|
|
||||||
|
// 2. 创建服务 (同时支持 IPv4 + IPv6)
|
||||||
|
let region = Ip2Region(v4Cfg, v6Cfg)
|
||||||
|
|
||||||
|
// 3. 查询 IPv4
|
||||||
|
let r1 = region.search("220.181.108.183")
|
||||||
|
println(" IPv4: 220.181.108.183 -> ${r1}")
|
||||||
|
|
||||||
|
// 4. 查询 IPv6
|
||||||
|
let r2 = region.search("2408:8266:100:1000::")
|
||||||
|
println(" IPv6: 2408:8266:100:1000:: -> ${r2}")
|
||||||
|
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// 方式三: 单版本服务 (仅 IPv4 或 仅 IPv6)
|
||||||
|
// 使用 init(config, ipVersion) 构造函数
|
||||||
|
// ============================================================
|
||||||
|
func demoSingleVersion(v4Path: String, v6Path: String) {
|
||||||
|
println("\n=== 方式三: 单版本服务 ===")
|
||||||
|
|
||||||
|
// IPv4 only
|
||||||
|
let v4Cfg = Config(ContentBuff, IPv4, v4Path, 3)
|
||||||
|
let v4Region = Ip2Region(v4Cfg, IPv4)
|
||||||
|
let r4 = v4Region.search("220.181.108.183")
|
||||||
|
println(" IPv4 only: 220.181.108.183 -> ${r4}")
|
||||||
|
v4Region.close()
|
||||||
|
|
||||||
|
// IPv6 only
|
||||||
|
let v6Cfg = Config(ContentBuff, IPv6, v6Path, 3)
|
||||||
|
let v6Region = Ip2Region(v6Cfg, IPv6)
|
||||||
|
let r6 = v6Region.search("2408:8266:100:1000::")
|
||||||
|
println(" IPv6 only: 2408:8266:100:1000:: -> ${r6}")
|
||||||
|
v6Region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// 方式四: 使用简便工厂函数 (ip2region.service)
|
||||||
|
// 一行创建,默认 VectorIndex + 20 连接池
|
||||||
|
// ============================================================
|
||||||
|
func demoFactory(v4Path: String, v6Path: String) {
|
||||||
|
println("\n=== 方式四: 简便工厂函数 ===")
|
||||||
|
|
||||||
|
let region = newIp2Region(v4Path, v6Path)
|
||||||
|
let result = region.search("220.181.108.183")
|
||||||
|
println(" 220.181.108.183 -> ${result}")
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================
|
||||||
|
// 性能测试: 测试三种缓存策略的查询速度
|
||||||
|
// ============================================================
|
||||||
|
func bench(v4Path: String) {
|
||||||
|
println("\n=== 性能测试 (IPv4, 487k 条记录) ===")
|
||||||
|
|
||||||
|
let content = File.readFrom(Path(v4Path))
|
||||||
|
let header = newHeaderFromBytes(content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let vIndex = loadVectorIndex(content)
|
||||||
|
|
||||||
|
// 从数据文件读取测试 IP 列表
|
||||||
|
let srcPath = v4Path.replace("ip2region_v4.xdb", "ipv4_source.txt")
|
||||||
|
let srcContent = File.readFrom(Path(srcPath))
|
||||||
|
let lines = String.fromUtf8(srcContent).split("\n")
|
||||||
|
|
||||||
|
var ipList = ArrayList<Array<Byte>>()
|
||||||
|
for (line in lines) {
|
||||||
|
let trimmed = line.trimAsciiStart().trimAsciiEnd()
|
||||||
|
if (trimmed.size == 0) { continue }
|
||||||
|
let parts = trimmed.split("|")
|
||||||
|
if (parts.size < 3) { continue }
|
||||||
|
try { ipList.add(parseIP(parts[0])) } catch (_) { continue }
|
||||||
|
}
|
||||||
|
println(" 加载了 ${ipList.size} 个 IP")
|
||||||
|
|
||||||
|
// 测试 ContentBuff 模式
|
||||||
|
println("\n [ContentBuff 模式]")
|
||||||
|
let s1 = Searcher(version, content, v4Path)
|
||||||
|
benchSearcher(s1, ipList)
|
||||||
|
|
||||||
|
// 测试 VectorIndex 模式
|
||||||
|
println(" [VectorIndex 模式]")
|
||||||
|
let s2 = Searcher(version, v4Path, vIndex)
|
||||||
|
benchSearcher(s2, ipList)
|
||||||
|
|
||||||
|
// 测试 FileOnly 模式
|
||||||
|
println(" [FileOnly 模式]")
|
||||||
|
let s3 = Searcher(version, v4Path)
|
||||||
|
benchSearcher(s3, ipList)
|
||||||
|
|
||||||
|
s1.close(); s2.close(); s3.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func benchSearcher(searcher: Searcher, ipList: ArrayList<Array<Byte>>) {
|
||||||
|
let count = ipList.size
|
||||||
|
let start = MonoTime.now()
|
||||||
|
for (i in 0..count) {
|
||||||
|
let _ = searcher.search(ipList[i])
|
||||||
|
}
|
||||||
|
let elapsed = MonoTime.now() - start
|
||||||
|
let avg = elapsed.toMicroseconds() / count
|
||||||
|
println(" 总量: ${count}, 耗时: ${elapsed.toMilliseconds()} ms, 平均: ${avg} us/op")
|
||||||
|
}
|
||||||
|
|
||||||
|
main(args: Array<String>): Int64 {
|
||||||
|
var v4Path = "../../../data/ip2region_v4.xdb"
|
||||||
|
var v6Path = "../../../data/ip2region_v6.xdb"
|
||||||
|
|
||||||
|
var i = 0
|
||||||
|
while (i < args.size) {
|
||||||
|
if (args[i] == "--v4-db" && i + 1 < args.size) {
|
||||||
|
v4Path = args[i + 1]; i = i + 2
|
||||||
|
} else if (args[i] == "--v6-db" && i + 1 < args.size) {
|
||||||
|
v6Path = args[i + 1]; i = i + 2
|
||||||
|
} else if (args[i] == "--help") {
|
||||||
|
printHelp(); return 0
|
||||||
|
} else { i = i + 1 }
|
||||||
|
}
|
||||||
|
|
||||||
|
println("ip2region Cangjie 使用示例")
|
||||||
|
println("xdb 路径: ${v4Path}")
|
||||||
|
|
||||||
|
demoXdbSearcher(v4Path, v6Path)
|
||||||
|
demoIp2Region(v4Path, v6Path)
|
||||||
|
demoSingleVersion(v4Path, v6Path)
|
||||||
|
demoFactory(v4Path, v6Path)
|
||||||
|
bench(v4Path)
|
||||||
|
|
||||||
|
println("\n完成!")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,225 @@
|
||||||
|
package ip2region
|
||||||
|
|
||||||
|
import std.fs.*
|
||||||
|
import std.time.*
|
||||||
|
import std.env.*
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
func printHelp() {
|
||||||
|
println("ip2region xdb searcher")
|
||||||
|
println("Usage: ip2region [command] [command options]")
|
||||||
|
println("Command:")
|
||||||
|
println(" search search input test")
|
||||||
|
println(" bench search bench test")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runSearch(args: Array<String>) {
|
||||||
|
var v4DbPath = "../../data/ip2region_v4.xdb"
|
||||||
|
var v6DbPath = "../../data/ip2region_v6.xdb"
|
||||||
|
var v4CachePolicy = "vectorIndex"
|
||||||
|
var v6CachePolicy = "vectorIndex"
|
||||||
|
var showHelp = false
|
||||||
|
|
||||||
|
var i = 1 // skip command name
|
||||||
|
while (i < args.size) {
|
||||||
|
let arg = args[i]
|
||||||
|
if (arg == "--v4-db" && i + 1 < args.size) {
|
||||||
|
v4DbPath = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else if (arg == "--v6-db" && i + 1 < args.size) {
|
||||||
|
v6DbPath = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else if (arg == "--v4-cache-policy" && i + 1 < args.size) {
|
||||||
|
v4CachePolicy = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else if (arg == "--v6-cache-policy" && i + 1 < args.size) {
|
||||||
|
v6CachePolicy = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else if (arg == "--help") {
|
||||||
|
showHelp = true
|
||||||
|
i = i + 1
|
||||||
|
} else {
|
||||||
|
i = i + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (showHelp) {
|
||||||
|
println("ip2region search [command options]")
|
||||||
|
println("options:")
|
||||||
|
println(" --v4-db <path> ip2region v4 binary xdb file path")
|
||||||
|
println(" --v4-cache-policy <mode> v4 cache policy: file/vectorIndex/content")
|
||||||
|
println(" --v6-db <path> ip2region v6 binary xdb file path")
|
||||||
|
println(" --v6-cache-policy <mode> v6 cache policy: file/vectorIndex/content")
|
||||||
|
println(" --help print this help")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load v4 xdb
|
||||||
|
print("Loading v4 xdb ... ")
|
||||||
|
let v4Content = File.readFrom(Path(v4DbPath))
|
||||||
|
let v4Header = newHeaderFromBytes(v4Content)
|
||||||
|
let v4Version = versionFromHeader(v4Header)
|
||||||
|
let v4Searcher = createSearcher(v4Version, v4Content, v4DbPath, v4CachePolicy)
|
||||||
|
println("done")
|
||||||
|
|
||||||
|
// Load v6 xdb
|
||||||
|
print("Loading v6 xdb ... ")
|
||||||
|
let v6Content = File.readFrom(Path(v6DbPath))
|
||||||
|
let v6Header = newHeaderFromBytes(v6Content)
|
||||||
|
let v6Version = versionFromHeader(v6Header)
|
||||||
|
let v6Searcher = createSearcher(v6Version, v6Content, v6DbPath, v6CachePolicy)
|
||||||
|
println("done")
|
||||||
|
|
||||||
|
println("ip2region search service test program")
|
||||||
|
println("type 'quit' to exit")
|
||||||
|
|
||||||
|
let reader = getStdIn()
|
||||||
|
while (true) {
|
||||||
|
print("ip2region>> ")
|
||||||
|
let lineOpt = reader.readln()
|
||||||
|
if (lineOpt == None) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
let line = lineOpt.getOrThrow()
|
||||||
|
if (line == "quit" || line == "exit") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if (line.size == 0) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let start = MonoTime.now()
|
||||||
|
let region = searchIP(line, v4Searcher, v6Searcher)
|
||||||
|
let elapsed = MonoTime.now() - start
|
||||||
|
println("${line} -> ${region} (took: ${elapsed.toMicroseconds()} us)")
|
||||||
|
}
|
||||||
|
|
||||||
|
v4Searcher.close()
|
||||||
|
v6Searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func runBench(args: Array<String>) {
|
||||||
|
var dbFile = ""
|
||||||
|
var srcFile = ""
|
||||||
|
var cachePolicy = "content"
|
||||||
|
|
||||||
|
var i = 1 // skip command name
|
||||||
|
while (i < args.size) {
|
||||||
|
let arg = args[i]
|
||||||
|
if (arg == "--db" && i + 1 < args.size) {
|
||||||
|
dbFile = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else if (arg == "--src" && i + 1 < args.size) {
|
||||||
|
srcFile = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else if (arg == "--cache-policy" && i + 1 < args.size) {
|
||||||
|
cachePolicy = args[i + 1]
|
||||||
|
i = i + 2
|
||||||
|
} else {
|
||||||
|
i = i + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dbFile == "" || srcFile == "") {
|
||||||
|
println("ip2region bench [command options]")
|
||||||
|
println("options:")
|
||||||
|
println(" --db <path> ip2region binary xdb file path")
|
||||||
|
println(" --src <path> source ip text file path")
|
||||||
|
println(" --cache-policy <mode> cache policy: file/vectorIndex/content")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
println("Loading xdb from: ${dbFile}")
|
||||||
|
let content = File.readFrom(Path(dbFile))
|
||||||
|
let header = newHeaderFromBytes(content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let searcher = createSearcher(version, content, dbFile, cachePolicy)
|
||||||
|
|
||||||
|
println("Loading source data from: ${srcFile}")
|
||||||
|
let srcContent = File.readFrom(Path(srcFile))
|
||||||
|
let srcStr = String.fromUtf8(srcContent)
|
||||||
|
let lines = srcStr.split("\n")
|
||||||
|
|
||||||
|
var count: Int64 = 0
|
||||||
|
var totalCost: Int64 = 0
|
||||||
|
let tStart = MonoTime.now()
|
||||||
|
|
||||||
|
for (line in lines) {
|
||||||
|
let trimmed = line.trimAsciiStart().trimAsciiEnd()
|
||||||
|
if (trimmed.size == 0) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let parts = trimmed.split("|")
|
||||||
|
if (parts.size < 3) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
let ipBytes = parseIP(parts[0])
|
||||||
|
let t0 = MonoTime.now()
|
||||||
|
let _ = searcher.search(ipBytes)
|
||||||
|
let t1 = MonoTime.now()
|
||||||
|
totalCost = totalCost + (t1 - t0).toNanoseconds()
|
||||||
|
count = count + 1
|
||||||
|
|
||||||
|
if (count % 10000 == 0) {
|
||||||
|
print(".")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let tEnd = MonoTime.now()
|
||||||
|
let totalTime = tEnd - tStart
|
||||||
|
|
||||||
|
println("")
|
||||||
|
println("Bench finished:")
|
||||||
|
println(" cachePolicy: ${cachePolicy}")
|
||||||
|
println(" total: ${count}")
|
||||||
|
println(" took: ${totalTime.toMilliseconds()} ms")
|
||||||
|
if (count > 0) {
|
||||||
|
println(" avg: ${totalCost / count / 1000} us/op")
|
||||||
|
}
|
||||||
|
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func createSearcher(version: Version, content: Array<Byte>, dbPath: String, cachePolicy: String): Searcher {
|
||||||
|
if (cachePolicy == "content") {
|
||||||
|
println("Using content buffer mode")
|
||||||
|
return Searcher(version, content, dbPath)
|
||||||
|
} else if (cachePolicy == "vectorIndex") {
|
||||||
|
println("Using vector index mode")
|
||||||
|
let vIndex = loadVectorIndex(content)
|
||||||
|
return Searcher(version, dbPath, vIndex)
|
||||||
|
} else {
|
||||||
|
println("Using file only mode")
|
||||||
|
return Searcher(version, dbPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func searchIP(ipStr: String, v4Searcher: Searcher, v6Searcher: Searcher): String {
|
||||||
|
let ipBytes = parseIP(ipStr)
|
||||||
|
if (ipBytes.size == 4) {
|
||||||
|
return v4Searcher.search(ipBytes)
|
||||||
|
} else if (ipBytes.size == 16) {
|
||||||
|
return v6Searcher.search(ipBytes)
|
||||||
|
}
|
||||||
|
return "Invalid IP"
|
||||||
|
}
|
||||||
|
|
||||||
|
main(args: Array<String>): Int64 {
|
||||||
|
if (args.size == 0) {
|
||||||
|
printHelp()
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
let cmd = args[0]
|
||||||
|
if (cmd == "search") {
|
||||||
|
runSearch(args)
|
||||||
|
} else if (cmd == "bench") {
|
||||||
|
runBench(args)
|
||||||
|
} else {
|
||||||
|
printHelp()
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
package ip2region.service
|
||||||
|
|
||||||
|
import std.fs.*
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
// Cache policy constants - match Searcher modes
|
||||||
|
public let FileOnly: Int64 = 0
|
||||||
|
public let VectorIndex: Int64 = 1
|
||||||
|
public let ContentBuff: Int64 = 2
|
||||||
|
|
||||||
|
// Config holds xdb file configuration and pre-loaded data for creating Searcher instances
|
||||||
|
public class Config {
|
||||||
|
public let cachePolicy: Int64
|
||||||
|
public let ipVersion: Version
|
||||||
|
public let xdbPath: String
|
||||||
|
public let header: Header
|
||||||
|
public let vIndex: Array<Byte>
|
||||||
|
public let cBuffer: Array<Byte>
|
||||||
|
public let searchers: Int64
|
||||||
|
|
||||||
|
public init(cachePolicy: Int64, ipVersion: Version, xdbPath: String, searchers: Int64) {
|
||||||
|
if (searchers <= 0) {
|
||||||
|
throw Exception("searchers must be > 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
this.cachePolicy = cachePolicy
|
||||||
|
this.ipVersion = ipVersion
|
||||||
|
this.xdbPath = xdbPath
|
||||||
|
this.searchers = searchers
|
||||||
|
|
||||||
|
let content = File.readFrom(Path(xdbPath))
|
||||||
|
this.header = newHeaderFromBytes(content)
|
||||||
|
|
||||||
|
// Verify IP version matches
|
||||||
|
let detectedVersion = versionFromHeader(this.header)
|
||||||
|
if (detectedVersion.id != ipVersion.id) {
|
||||||
|
throw Exception("xdb file IP version mismatch: expected ${ipVersion.name}, got ${detectedVersion.name}")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cachePolicy == VectorIndex) {
|
||||||
|
this.vIndex = loadVectorIndex(content)
|
||||||
|
this.cBuffer = Array<Byte>(0, repeat: 0)
|
||||||
|
} else if (cachePolicy == ContentBuff) {
|
||||||
|
this.vIndex = Array<Byte>(0, repeat: 0)
|
||||||
|
this.cBuffer = content
|
||||||
|
} else {
|
||||||
|
this.vIndex = Array<Byte>(0, repeat: 0)
|
||||||
|
this.cBuffer = Array<Byte>(0, repeat: 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public init(cachePolicy: Int64, ipVersion: Version, xdbPath: String) {
|
||||||
|
this(cachePolicy, ipVersion, xdbPath, 20)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Static factories for IPv4 and IPv6
|
||||||
|
public func newV4Config(cachePolicy: Int64, xdbPath: String, searchers: Int64): Config {
|
||||||
|
return Config(cachePolicy, IPv4, xdbPath, searchers)
|
||||||
|
}
|
||||||
|
|
||||||
|
public func newV6Config(cachePolicy: Int64, xdbPath: String, searchers: Int64): Config {
|
||||||
|
return Config(cachePolicy, IPv6, xdbPath, searchers)
|
||||||
|
}
|
||||||
|
|
||||||
|
public func newV4Config(cachePolicy: Int64, xdbPath: String): Config {
|
||||||
|
return Config(cachePolicy, IPv4, xdbPath, 20)
|
||||||
|
}
|
||||||
|
|
||||||
|
public func newV6Config(cachePolicy: Int64, xdbPath: String): Config {
|
||||||
|
return Config(cachePolicy, IPv6, xdbPath, 20)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,161 @@
|
||||||
|
package ip2region.service
|
||||||
|
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
// Ip2Region is the high-level API for searching IP addresses.
|
||||||
|
// It supports both IPv4 and IPv6 with thread-safe access via SearcherPool.
|
||||||
|
public class Ip2Region {
|
||||||
|
let v4Pool: SearcherPool
|
||||||
|
let v6Pool: SearcherPool
|
||||||
|
let v4InMemSearcher: Searcher
|
||||||
|
let v6InMemSearcher: Searcher
|
||||||
|
let hasV4: Bool
|
||||||
|
let hasV6: Bool
|
||||||
|
let v4IsInMem: Bool
|
||||||
|
let v6IsInMem: Bool
|
||||||
|
|
||||||
|
// Create Ip2Region with both v4 and v6 configurations.
|
||||||
|
public init(v4Config: Config, v6Config: Config) {
|
||||||
|
// v4 setup
|
||||||
|
if (v4Config.cachePolicy == ContentBuff) {
|
||||||
|
this.hasV4 = true
|
||||||
|
this.v4IsInMem = true
|
||||||
|
this.v4InMemSearcher = Searcher(v4Config.ipVersion, v4Config.cBuffer, v4Config.xdbPath)
|
||||||
|
this.v4Pool = SearcherPool(v4Config)
|
||||||
|
} else {
|
||||||
|
this.hasV4 = true
|
||||||
|
this.v4IsInMem = false
|
||||||
|
this.v4InMemSearcher = Searcher(v4Config.ipVersion, Array<Byte>(0, repeat: 0), v4Config.xdbPath)
|
||||||
|
this.v4Pool = SearcherPool(v4Config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// v6 setup
|
||||||
|
if (v6Config.cachePolicy == ContentBuff) {
|
||||||
|
this.hasV6 = true
|
||||||
|
this.v6IsInMem = true
|
||||||
|
this.v6InMemSearcher = Searcher(v6Config.ipVersion, v6Config.cBuffer, v6Config.xdbPath)
|
||||||
|
this.v6Pool = SearcherPool(v6Config)
|
||||||
|
} else {
|
||||||
|
this.hasV6 = true
|
||||||
|
this.v6IsInMem = false
|
||||||
|
this.v6InMemSearcher = Searcher(v6Config.ipVersion, Array<Byte>(0, repeat: 0), v6Config.xdbPath)
|
||||||
|
this.v6Pool = SearcherPool(v6Config)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single-version: provide Config and IP version.
|
||||||
|
// Examples:
|
||||||
|
// Ip2Region(v4Config, IPv4) — IPv4 only
|
||||||
|
// Ip2Region(v6Config, IPv6) — IPv6 only
|
||||||
|
public init(config: Config, ipVersion: Version) {
|
||||||
|
if (ipVersion.id == IPv4VersionNo) {
|
||||||
|
this.hasV4 = true
|
||||||
|
this.v4IsInMem = (config.cachePolicy == ContentBuff)
|
||||||
|
if (config.cachePolicy == ContentBuff) {
|
||||||
|
this.v4InMemSearcher = Searcher(config.ipVersion, config.cBuffer, config.xdbPath)
|
||||||
|
} else {
|
||||||
|
this.v4InMemSearcher = Searcher(config.ipVersion, Array<Byte>(0, repeat: 0), config.xdbPath)
|
||||||
|
}
|
||||||
|
this.v4Pool = SearcherPool(config)
|
||||||
|
// IPv6 disabled
|
||||||
|
this.hasV6 = false
|
||||||
|
this.v6IsInMem = false
|
||||||
|
this.v6InMemSearcher = Searcher(IPv4, Array<Byte>(0, repeat: 0), config.xdbPath)
|
||||||
|
this.v6Pool = SearcherPool(Config(ContentBuff, IPv4, config.xdbPath, 1))
|
||||||
|
} else {
|
||||||
|
this.hasV6 = true
|
||||||
|
this.v6IsInMem = (config.cachePolicy == ContentBuff)
|
||||||
|
if (config.cachePolicy == ContentBuff) {
|
||||||
|
this.v6InMemSearcher = Searcher(config.ipVersion, config.cBuffer, config.xdbPath)
|
||||||
|
} else {
|
||||||
|
this.v6InMemSearcher = Searcher(config.ipVersion, Array<Byte>(0, repeat: 0), config.xdbPath)
|
||||||
|
}
|
||||||
|
this.v6Pool = SearcherPool(config)
|
||||||
|
// IPv4 disabled
|
||||||
|
this.hasV4 = false
|
||||||
|
this.v4IsInMem = false
|
||||||
|
this.v4InMemSearcher = Searcher(IPv6, Array<Byte>(0, repeat: 0), config.xdbPath)
|
||||||
|
this.v4Pool = SearcherPool(Config(ContentBuff, IPv6, config.xdbPath, 1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search an IP string and return the region info.
|
||||||
|
public func search(ipStr: String): String {
|
||||||
|
try {
|
||||||
|
let ipBytes = parseIP(ipStr)
|
||||||
|
|
||||||
|
if (ipBytes.size == 4) {
|
||||||
|
return this.searchV4(ipBytes)
|
||||||
|
} else if (ipBytes.size == 16) {
|
||||||
|
return this.searchV6(ipBytes)
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
return "Invalid IP"
|
||||||
|
}
|
||||||
|
return "Invalid IP"
|
||||||
|
}
|
||||||
|
|
||||||
|
func searchV4(ip: Array<Byte>): String {
|
||||||
|
if (!this.hasV4) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if (this.v4IsInMem) {
|
||||||
|
return this.v4InMemSearcher.search(ip)
|
||||||
|
}
|
||||||
|
let searcher = this.v4Pool.borrow()
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
this.v4Pool.return_(searcher)
|
||||||
|
return region
|
||||||
|
}
|
||||||
|
|
||||||
|
func searchV6(ip: Array<Byte>): String {
|
||||||
|
if (!this.hasV6) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if (this.v6IsInMem) {
|
||||||
|
return this.v6InMemSearcher.search(ip)
|
||||||
|
}
|
||||||
|
let searcher = this.v6Pool.borrow()
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
this.v6Pool.return_(searcher)
|
||||||
|
return region
|
||||||
|
}
|
||||||
|
|
||||||
|
// close releases all resources.
|
||||||
|
public func close() {
|
||||||
|
if (this.hasV4) {
|
||||||
|
this.v4Pool.close()
|
||||||
|
}
|
||||||
|
if (this.hasV6) {
|
||||||
|
this.v6Pool.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// closeTimeout releases all resources with the specified timeout.
|
||||||
|
public func closeTimeout(timeout: Duration) {
|
||||||
|
if (this.hasV4) {
|
||||||
|
this.v4Pool.closeTimeout(timeout)
|
||||||
|
}
|
||||||
|
if (this.hasV6) {
|
||||||
|
this.v6Pool.closeTimeout(timeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create an Ip2Region instance using xdb file paths.
|
||||||
|
// Uses VectorIndex cache policy with 20 searchers by default.
|
||||||
|
public func newIp2Region(v4XdbPath: String, v6XdbPath: String): Ip2Region {
|
||||||
|
let v4Config = Config(VectorIndex, IPv4, v4XdbPath, 20)
|
||||||
|
let v6Config = Config(VectorIndex, IPv6, v6XdbPath, 20)
|
||||||
|
return Ip2Region(v4Config, v6Config)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create an IPv4-only Ip2Region service.
|
||||||
|
public func newIp2RegionV4(config: Config): Ip2Region {
|
||||||
|
return Ip2Region(config, IPv4)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create an IPv6-only Ip2Region service.
|
||||||
|
public func newIp2RegionV6(config: Config): Ip2Region {
|
||||||
|
return Ip2Region(config, IPv6)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,111 @@
|
||||||
|
package ip2region.service
|
||||||
|
|
||||||
|
import std.sync.*
|
||||||
|
import std.core.*
|
||||||
|
import std.time.*
|
||||||
|
import std.collection.concurrent.*
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
// SearcherPool is a thread-safe pool of xdb.Searcher instances.
|
||||||
|
// Uses Semaphore for backpressure and ConcurrentLinkedQueue for storage.
|
||||||
|
public class SearcherPool {
|
||||||
|
let config: Config
|
||||||
|
let queue: ConcurrentLinkedQueue<Searcher>
|
||||||
|
let semaphore: Semaphore
|
||||||
|
let closing: AtomicBool
|
||||||
|
let loanCnt: AtomicInt64
|
||||||
|
|
||||||
|
public init(config: Config) {
|
||||||
|
this.config = config
|
||||||
|
this.queue = ConcurrentLinkedQueue<Searcher>()
|
||||||
|
this.semaphore = Semaphore(config.searchers)
|
||||||
|
this.closing = AtomicBool(false)
|
||||||
|
this.loanCnt = AtomicInt64(0)
|
||||||
|
|
||||||
|
let poolSize = config.searchers
|
||||||
|
for (_ in 0..poolSize) {
|
||||||
|
let searcher = createPoolSearcher(config)
|
||||||
|
this.queue.add(searcher)
|
||||||
|
this.semaphore.release(amount: 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// borrow takes a Searcher from the pool, blocking until one is available.
|
||||||
|
// Throws if the pool is closing or closed.
|
||||||
|
public func borrow(): Searcher {
|
||||||
|
while (!this.closing.load()) {
|
||||||
|
if (this.semaphore.tryAcquire(amount: 1)) {
|
||||||
|
let opt = this.queue.remove()
|
||||||
|
if (opt.isNone()) {
|
||||||
|
this.semaphore.release(amount: 1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
let s = opt.getOrThrow()
|
||||||
|
this.loanCnt.fetchAdd(1)
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
sleep(Duration.millisecond * 5)
|
||||||
|
}
|
||||||
|
throw Exception("SearcherPool is closing")
|
||||||
|
}
|
||||||
|
|
||||||
|
// return_ returns a Searcher to the pool, or closes it if the pool is shutting down.
|
||||||
|
public func return_(searcher: Searcher) {
|
||||||
|
if (this.closing.load()) {
|
||||||
|
searcher.close()
|
||||||
|
} else {
|
||||||
|
this.queue.add(searcher)
|
||||||
|
this.semaphore.release(amount: 1)
|
||||||
|
}
|
||||||
|
this.loanCnt.fetchSub(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// close closes the pool with a default 10 second timeout.
|
||||||
|
public func close() {
|
||||||
|
this.closeTimeout(Duration.second * 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
// closeTimeout closes the pool with the specified timeout.
|
||||||
|
public func closeTimeout(timeout: Duration) {
|
||||||
|
this.closing.store(true)
|
||||||
|
|
||||||
|
// Drain the queue: acquire all permits and close searchers
|
||||||
|
while (true) {
|
||||||
|
if (!this.semaphore.tryAcquire(amount: 1)) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
let opt = this.queue.remove()
|
||||||
|
if (opt.isSome()) {
|
||||||
|
opt.getOrThrow().close()
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for outstanding loans to return (with timeout)
|
||||||
|
let deadline = MonoTime.now() + timeout
|
||||||
|
while (this.loanCnt.load() > 0) {
|
||||||
|
if (MonoTime.now() >= deadline) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sleep(Duration.millisecond * 10)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public func getLoanCount(): Int64 {
|
||||||
|
return this.loanCnt.load()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper to create a Searcher based on Config's cache policy
|
||||||
|
func createPoolSearcher(config: Config): Searcher {
|
||||||
|
if (config.cachePolicy == ContentBuff) {
|
||||||
|
return Searcher(config.ipVersion, config.cBuffer, config.xdbPath)
|
||||||
|
} else if (config.cachePolicy == VectorIndex) {
|
||||||
|
return Searcher(config.ipVersion, config.xdbPath, config.vIndex)
|
||||||
|
} else {
|
||||||
|
return Searcher(config.ipVersion, config.xdbPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
package ip2region.tests.service
|
||||||
|
|
||||||
|
import ip2region.xdb.*
|
||||||
|
import ip2region.service.*
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class ConfigV4Tests {
|
||||||
|
@TestCase
|
||||||
|
func testCreateFileOnly() {
|
||||||
|
let cfg = Config(FileOnly, IPv4, "../../data/ip2region_v4.xdb", 5)
|
||||||
|
if (cfg.cachePolicy != FileOnly) { throw Exception("wrong cache policy") }
|
||||||
|
if (cfg.searchers != 5) { throw Exception("wrong searchers count") }
|
||||||
|
if (cfg.ipVersion.id != IPv4VersionNo) { throw Exception("wrong version") }
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testCreateVectorIndex() {
|
||||||
|
let cfg = Config(VectorIndex, IPv4, "../../data/ip2region_v4.xdb", 10)
|
||||||
|
if (cfg.vIndex.size == 0) { throw Exception("vIndex should be loaded") }
|
||||||
|
let expectedSize = VectorIndexRows * VectorIndexCols * VectorIndexSize
|
||||||
|
if (cfg.vIndex.size != expectedSize) {
|
||||||
|
throw Exception("vIndex size mismatch: ${cfg.vIndex.size} vs ${expectedSize}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testCreateContentBuff() {
|
||||||
|
let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 3)
|
||||||
|
if (cfg.cBuffer.size == 0) { throw Exception("cBuffer should be loaded") }
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testFactoryV4() {
|
||||||
|
let cfg = newV4Config(VectorIndex, "../../data/ip2region_v4.xdb", 8)
|
||||||
|
if (cfg.ipVersion.id != IPv4VersionNo) { throw Exception("should be IPv4") }
|
||||||
|
if (cfg.searchers != 8) { throw Exception("wrong searchers") }
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testFactoryV6() {
|
||||||
|
let cfg = newV6Config(VectorIndex, "../../data/ip2region_v6.xdb", 8)
|
||||||
|
if (cfg.ipVersion.id != IPv6VersionNo) { throw Exception("should be IPv6") }
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testFactoryDefaultSearchers() {
|
||||||
|
let cfg = newV4Config(ContentBuff, "../../data/ip2region_v4.xdb")
|
||||||
|
if (cfg.searchers != 20) { throw Exception("default searchers should be 20") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
package ip2region.tests.service
|
||||||
|
|
||||||
|
import ip2region.xdb.*
|
||||||
|
import ip2region.service.*
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class Ip2RegionBasicTests {
|
||||||
|
@TestCase
|
||||||
|
func testSearchWithContentBuff() {
|
||||||
|
let v4Cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 1)
|
||||||
|
let v6Cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 1)
|
||||||
|
let region = Ip2Region(v4Cfg, v6Cfg)
|
||||||
|
let r1 = region.search("220.181.108.183")
|
||||||
|
if (r1.size == 0) {
|
||||||
|
throw Exception("v4 search should return result")
|
||||||
|
}
|
||||||
|
let r2 = region.search("2408:8266:100:1000::")
|
||||||
|
if (r2.size == 0) {
|
||||||
|
throw Exception("v6 search should return result")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchWithVectorIndex() {
|
||||||
|
let v4Cfg = Config(VectorIndex, IPv4, "../../data/ip2region_v4.xdb", 5)
|
||||||
|
let v6Cfg = Config(VectorIndex, IPv6, "../../data/ip2region_v6.xdb", 5)
|
||||||
|
let region = Ip2Region(v4Cfg, v6Cfg)
|
||||||
|
let r = region.search("220.181.108.183")
|
||||||
|
if (r.size == 0) {
|
||||||
|
throw Exception("v4 search should return result")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testInvalidIP() {
|
||||||
|
let v4Cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 1)
|
||||||
|
let v6Cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 1)
|
||||||
|
let region = Ip2Region(v4Cfg, v6Cfg)
|
||||||
|
let r = region.search("invalid")
|
||||||
|
if (r != "Invalid IP") {
|
||||||
|
throw Exception("expected 'Invalid IP', got '${r}'")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testClose() {
|
||||||
|
let v4Cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 1)
|
||||||
|
let v6Cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 1)
|
||||||
|
let region = Ip2Region(v4Cfg, v6Cfg)
|
||||||
|
region.search("220.181.108.183")
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testV4OnlyInit() {
|
||||||
|
let cfg = Config(VectorIndex, IPv4, "../../data/ip2region_v4.xdb", 3)
|
||||||
|
let region = Ip2Region(cfg, IPv4)
|
||||||
|
let r = region.search("220.181.108.183")
|
||||||
|
if (r.size == 0) {
|
||||||
|
throw Exception("v4-only region should find v4 IP")
|
||||||
|
}
|
||||||
|
let r6 = region.search("2408:8266:100:1000::")
|
||||||
|
if (r6.size != 0) {
|
||||||
|
throw Exception("v4-only region should not find v6 IP")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testV6OnlyInit() {
|
||||||
|
let cfg = Config(VectorIndex, IPv6, "../../data/ip2region_v6.xdb", 3)
|
||||||
|
let region = Ip2Region(cfg, IPv6)
|
||||||
|
let r = region.search("2408:8266:100:1000::")
|
||||||
|
if (r.size == 0) {
|
||||||
|
throw Exception("v6-only region should find v6 IP")
|
||||||
|
}
|
||||||
|
let r4 = region.search("220.181.108.183")
|
||||||
|
if (r4.size != 0) {
|
||||||
|
throw Exception("v6-only region should not find v4 IP")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testNewV4Factory() {
|
||||||
|
let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 2)
|
||||||
|
let region = newIp2RegionV4(cfg)
|
||||||
|
let r = region.search("220.181.108.183")
|
||||||
|
if (r.size == 0) {
|
||||||
|
throw Exception("newIp2RegionV4 should work")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testNewV6Factory() {
|
||||||
|
let cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 2)
|
||||||
|
let region = newIp2RegionV6(cfg)
|
||||||
|
let r = region.search("2408:8266:100:1000::")
|
||||||
|
if (r.size == 0) {
|
||||||
|
throw Exception("newIp2RegionV6 should work")
|
||||||
|
}
|
||||||
|
region.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
package ip2region.tests.service
|
||||||
|
|
||||||
|
import ip2region.xdb.*
|
||||||
|
import ip2region.service.*
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class SearcherPoolTests {
|
||||||
|
@TestCase
|
||||||
|
func testBorrowAndReturn() {
|
||||||
|
let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 3)
|
||||||
|
let pool = SearcherPool(cfg)
|
||||||
|
let s = pool.borrow()
|
||||||
|
if (pool.getLoanCount() != 1) {
|
||||||
|
throw Exception("loan count should be 1, got ${pool.getLoanCount()}")
|
||||||
|
}
|
||||||
|
pool.return_(s)
|
||||||
|
if (pool.getLoanCount() != 0) {
|
||||||
|
throw Exception("loan count should be 0 after return")
|
||||||
|
}
|
||||||
|
pool.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testBorrowAll() {
|
||||||
|
let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 5)
|
||||||
|
let pool = SearcherPool(cfg)
|
||||||
|
let s1 = pool.borrow()
|
||||||
|
let s2 = pool.borrow()
|
||||||
|
let s3 = pool.borrow()
|
||||||
|
let s4 = pool.borrow()
|
||||||
|
let s5 = pool.borrow()
|
||||||
|
if (pool.getLoanCount() != 5) {
|
||||||
|
throw Exception("should have 5 loans, got ${pool.getLoanCount()}")
|
||||||
|
}
|
||||||
|
pool.return_(s5)
|
||||||
|
pool.return_(s4)
|
||||||
|
pool.return_(s3)
|
||||||
|
pool.return_(s2)
|
||||||
|
pool.return_(s1)
|
||||||
|
if (pool.getLoanCount() != 0) {
|
||||||
|
throw Exception("all should be returned, got ${pool.getLoanCount()}")
|
||||||
|
}
|
||||||
|
pool.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testClosePool() {
|
||||||
|
let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 2)
|
||||||
|
let pool = SearcherPool(cfg)
|
||||||
|
let s = pool.borrow()
|
||||||
|
pool.return_(s)
|
||||||
|
pool.close()
|
||||||
|
pool.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
package ip2region.tests
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
package ip2region.tests.xdb
|
||||||
|
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class HeaderTests {
|
||||||
|
func makeHeaderBytes(): Array<Byte> {
|
||||||
|
let buf = Array<Byte>(256, repeat: 0)
|
||||||
|
var v: Byte
|
||||||
|
v = 3; buf[0] = v; v = 0; buf[1] = v
|
||||||
|
v = 1; buf[2] = v; v = 0; buf[3] = v
|
||||||
|
v = 0xA0; buf[4] = v; v = 0x86; buf[5] = v; v = 0x01; buf[6] = v; v = 0x00; buf[7] = v
|
||||||
|
v = 0xE8; buf[8] = v; v = 0x03; buf[9] = v; v = 0x00; buf[10] = v; v = 0x00; buf[11] = v
|
||||||
|
v = 0xD0; buf[12] = v; v = 0x07; buf[13] = v; v = 0x00; buf[14] = v; v = 0x00; buf[15] = v
|
||||||
|
v = 4; buf[16] = v; v = 0; buf[17] = v
|
||||||
|
v = 0; buf[18] = v; v = 0; buf[19] = v
|
||||||
|
return buf
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testNewHeaderFromBytes() {
|
||||||
|
let buf = makeHeaderBytes()
|
||||||
|
let h = newHeaderFromBytes(buf)
|
||||||
|
if (h.version != UInt16(3)) {
|
||||||
|
throw Exception("expected version 3, got ${h.version}")
|
||||||
|
}
|
||||||
|
if (h.ipVersion != UInt16(4)) {
|
||||||
|
throw Exception("expected ipVersion 4, got ${h.ipVersion}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testHeaderToString() {
|
||||||
|
let buf = makeHeaderBytes()
|
||||||
|
let h = newHeaderFromBytes(buf)
|
||||||
|
let s = h.toString()
|
||||||
|
if (s.size == 0) {
|
||||||
|
throw Exception("toString should not be empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testVersionFromHeaderV4() {
|
||||||
|
let buf = makeHeaderBytes()
|
||||||
|
let h = newHeaderFromBytes(buf)
|
||||||
|
let v = versionFromHeader(h)
|
||||||
|
if (v.id != IPv4VersionNo) {
|
||||||
|
throw Exception("expected IPv4, got ${v.name}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testVersionFromHeaderV6() {
|
||||||
|
let buf = makeHeaderBytes()
|
||||||
|
var v: Byte = 6; buf[16] = v
|
||||||
|
let h = newHeaderFromBytes(buf)
|
||||||
|
let ver = versionFromHeader(h)
|
||||||
|
if (ver.id != IPv6VersionNo) {
|
||||||
|
throw Exception("expected IPv6, got ${ver.name}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testVersionFromHeader20() {
|
||||||
|
let buf = makeHeaderBytes()
|
||||||
|
var v: Byte = 2; buf[0] = v
|
||||||
|
v = 0; buf[16] = v
|
||||||
|
let h = newHeaderFromBytes(buf)
|
||||||
|
let ver = versionFromHeader(h)
|
||||||
|
if (ver.id != IPv4VersionNo) {
|
||||||
|
throw Exception("expected IPv4 for v2.0")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testHeaderConstants() {
|
||||||
|
if (HeaderInfoLength != 256) { throw Exception("expected 256") }
|
||||||
|
if (VectorIndexRows != 256) { throw Exception("expected 256") }
|
||||||
|
if (VectorIndexSize != 8) { throw Exception("expected 8") }
|
||||||
|
if (IPv4SegmentIndexSize != 14) { throw Exception("expected 14") }
|
||||||
|
if (IPv6SegmentIndexSize != 38) { throw Exception("expected 38") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,132 @@
|
||||||
|
package ip2region.tests.xdb
|
||||||
|
|
||||||
|
import std.fs.*
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class SearcherV4Tests {
|
||||||
|
var v4Content: Array<Byte> = Array<Byte>(0, repeat: 0)
|
||||||
|
var initialized: Bool = false
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
func setup() {
|
||||||
|
this.v4Content = File.readFrom(Path("../../data/ip2region_v4.xdb"))
|
||||||
|
this.initialized = true
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchContentMode() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v4Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let searcher = Searcher(version, this.v4Content, "../../data/ip2region_v4.xdb")
|
||||||
|
let ip = parseIP("220.181.108.183")
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
if (region.size == 0) {
|
||||||
|
throw Exception("should find region for known IP")
|
||||||
|
}
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchVectorIndexMode() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v4Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let vIndex = loadVectorIndex(this.v4Content)
|
||||||
|
let searcher = Searcher(version, "../../data/ip2region_v4.xdb", vIndex)
|
||||||
|
let ip = parseIP("220.181.108.183")
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
if (region.size == 0) {
|
||||||
|
throw Exception("should find region for known IP")
|
||||||
|
}
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchFileMode() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v4Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let searcher = Searcher(version, "../../data/ip2region_v4.xdb")
|
||||||
|
let ip = parseIP("220.181.108.183")
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
if (region.size == 0) {
|
||||||
|
throw Exception("should find region for known IP")
|
||||||
|
}
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testAllModesSameResult() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v4Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let vIndex = loadVectorIndex(this.v4Content)
|
||||||
|
let s1 = Searcher(version, this.v4Content, "../../data/ip2region_v4.xdb")
|
||||||
|
let s2 = Searcher(version, "../../data/ip2region_v4.xdb", vIndex)
|
||||||
|
let s3 = Searcher(version, "../../data/ip2region_v4.xdb")
|
||||||
|
let ip = parseIP("1.2.3.4")
|
||||||
|
let r1 = s1.search(ip)
|
||||||
|
let r2 = s2.search(ip)
|
||||||
|
let r3 = s3.search(ip)
|
||||||
|
s1.close(); s2.close(); s3.close()
|
||||||
|
if (r1 != r2 || r2 != r3) {
|
||||||
|
throw Exception("all modes should return same result")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchByString() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v4Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let searcher = Searcher(version, this.v4Content, "../../data/ip2region_v4.xdb")
|
||||||
|
let region = searcher.searchByString("220.181.108.183")
|
||||||
|
if (region.size == 0) {
|
||||||
|
throw Exception("searchByString should find region")
|
||||||
|
}
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class SearcherV6Tests {
|
||||||
|
var v6Content: Array<Byte> = Array<Byte>(0, repeat: 0)
|
||||||
|
var initialized: Bool = false
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
func setup() {
|
||||||
|
this.v6Content = File.readFrom(Path("../../data/ip2region_v6.xdb"))
|
||||||
|
this.initialized = true
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchV6ContentMode() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v6Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let searcher = Searcher(version, this.v6Content, "../../data/ip2region_v6.xdb")
|
||||||
|
let ip = parseIP("2408:8266:100:1000::")
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
if (region.size == 0) {
|
||||||
|
throw Exception("should find region for known IPv6")
|
||||||
|
}
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testSearchV6VectorIndexMode() {
|
||||||
|
if (!this.initialized) { return }
|
||||||
|
let header = newHeaderFromBytes(this.v6Content)
|
||||||
|
let version = versionFromHeader(header)
|
||||||
|
let vIndex = loadVectorIndex(this.v6Content)
|
||||||
|
let searcher = Searcher(version, "../../data/ip2region_v6.xdb", vIndex)
|
||||||
|
let ip = parseIP("2408:8266:100:1000::")
|
||||||
|
let region = searcher.search(ip)
|
||||||
|
if (region.size == 0) {
|
||||||
|
throw Exception("should find region for known IPv6")
|
||||||
|
}
|
||||||
|
searcher.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,133 @@
|
||||||
|
package ip2region.tests.xdb
|
||||||
|
|
||||||
|
import ip2region.xdb.*
|
||||||
|
|
||||||
|
@Test
|
||||||
|
class UtilTests {
|
||||||
|
@TestCase
|
||||||
|
func testParseIPv4() {
|
||||||
|
let ip = parseIP("1.2.3.4")
|
||||||
|
if (ip.size != 4) {
|
||||||
|
throw Exception("expected 4 bytes, got ${ip.size}")
|
||||||
|
}
|
||||||
|
if (Int64(ip[0]) != 1 || Int64(ip[1]) != 2 || Int64(ip[2]) != 3 || Int64(ip[3]) != 4) {
|
||||||
|
throw Exception("IPv4 parse mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testParseIPv6() {
|
||||||
|
let ip = parseIP("::1")
|
||||||
|
if (ip.size != 16) {
|
||||||
|
throw Exception("expected 16 bytes for IPv6, got ${ip.size}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testIPCompareEqual() {
|
||||||
|
let a = Array<Byte>(4, repeat: 0)
|
||||||
|
var v: Byte = 1; a[0] = v; v = 2; a[1] = v; v = 3; a[2] = v; v = 4; a[3] = v
|
||||||
|
let b = Array<Byte>(4, repeat: 0)
|
||||||
|
v = 1; b[0] = v; v = 2; b[1] = v; v = 3; b[2] = v; v = 4; b[3] = v
|
||||||
|
let r = ipCompare(a, b)
|
||||||
|
if (r != 0) {
|
||||||
|
throw Exception("expected 0, got ${r}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testIPCompareLess() {
|
||||||
|
let a = Array<Byte>(4, repeat: 0)
|
||||||
|
var v: Byte = 1; a[0] = v; v = 2; a[1] = v; v = 3; a[2] = v; v = 4; a[3] = v
|
||||||
|
let b = Array<Byte>(4, repeat: 0)
|
||||||
|
v = 1; b[0] = v; v = 2; b[1] = v; v = 3; b[2] = v; v = 5; b[3] = v
|
||||||
|
let r = ipCompare(a, b)
|
||||||
|
if (r != -1) {
|
||||||
|
throw Exception("expected -1, got ${r}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testIPCompareGreater() {
|
||||||
|
let a = Array<Byte>(4, repeat: 0)
|
||||||
|
var v: Byte = 1; a[0] = v; v = 2; a[1] = v; v = 3; a[2] = v; v = 5; a[3] = v
|
||||||
|
let b = Array<Byte>(4, repeat: 0)
|
||||||
|
v = 1; b[0] = v; v = 2; b[1] = v; v = 3; b[2] = v; v = 4; b[3] = v
|
||||||
|
let r = ipCompare(a, b)
|
||||||
|
if (r != 1) {
|
||||||
|
throw Exception("expected 1, got ${r}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testReadLEUint32() {
|
||||||
|
let buf = Array<Byte>(4, repeat: 0)
|
||||||
|
var v: Byte = 0x78; buf[0] = v; v = 0x56; buf[1] = v; v = 0x34; buf[2] = v; v = 0x12; buf[3] = v
|
||||||
|
let val = readLEUint32(buf, 0)
|
||||||
|
let expected: Int64 = 0x12345678
|
||||||
|
if (val != expected) {
|
||||||
|
throw Exception("expected 0x12345678, got ${val}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testReadLEUint16() {
|
||||||
|
let buf = Array<Byte>(2, repeat: 0)
|
||||||
|
var v: Byte = 0x34; buf[0] = v; v = 0x12; buf[1] = v
|
||||||
|
let val = readLEUint16(buf, 0)
|
||||||
|
if (val != 0x1234) {
|
||||||
|
throw Exception("expected 0x1234, got ${val}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testIpToString() {
|
||||||
|
let ip = Array<Byte>(4, repeat: 0)
|
||||||
|
var v: Byte = 127; ip[0] = v; v = 0; ip[1] = v; v = 0; ip[2] = v; v = 1; ip[3] = v
|
||||||
|
let s = ipToString(ip)
|
||||||
|
if (s != "127.0.0.1") {
|
||||||
|
throw Exception("expected 127.0.0.1, got ${s}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testCompareIPv4() {
|
||||||
|
let ip = Array<Byte>(4, repeat: 0)
|
||||||
|
var v: Byte = 1; ip[0] = v; v = 2; ip[1] = v; v = 3; ip[2] = v; v = 4; ip[3] = v
|
||||||
|
let entry = Array<Byte>(8, repeat: 0)
|
||||||
|
v = 4; entry[0] = v; v = 3; entry[1] = v; v = 2; entry[2] = v; v = 1; entry[3] = v
|
||||||
|
v = 8; entry[4] = v; v = 7; entry[5] = v; v = 6; entry[6] = v; v = 5; entry[7] = v
|
||||||
|
let r = compareIPv4(ip, entry, 0)
|
||||||
|
if (r != 0) {
|
||||||
|
throw Exception("expected equal, got ${r}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testCompareIPv6() {
|
||||||
|
let ip = Array<Byte>(16, repeat: 0)
|
||||||
|
var v: Byte = 0x20; ip[0] = v; v = 0x01; ip[1] = v; v = 0x0d; ip[2] = v; v = 0xb8; ip[3] = v
|
||||||
|
let entry = Array<Byte>(16, repeat: 0)
|
||||||
|
v = 0x20; entry[0] = v; v = 0x01; entry[1] = v; v = 0x0d; entry[2] = v; v = 0xb8; entry[3] = v
|
||||||
|
let r = compareIPv6(ip, entry, 0)
|
||||||
|
if (r != 0) {
|
||||||
|
throw Exception("expected equal, got ${r}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@TestCase
|
||||||
|
func testLoadVectorIndex() {
|
||||||
|
let viSize = VectorIndexRows * VectorIndexCols * VectorIndexSize
|
||||||
|
let cBuff = Array<Byte>(HeaderInfoLength + viSize, repeat: 0)
|
||||||
|
var v: Byte = 0x42
|
||||||
|
cBuff[HeaderInfoLength] = v
|
||||||
|
let vi = loadVectorIndex(cBuff)
|
||||||
|
if (vi.size != viSize) {
|
||||||
|
throw Exception("expected size ${viSize}, got ${vi.size}")
|
||||||
|
}
|
||||||
|
let firstVal = Int64(vi[0])
|
||||||
|
if (firstVal != 0x42) {
|
||||||
|
throw Exception("expected first byte 0x42, got ${firstVal}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,77 @@
|
||||||
|
package ip2region.xdb
|
||||||
|
|
||||||
|
// xdb file format constants
|
||||||
|
public let HeaderInfoLength: Int64 = 256
|
||||||
|
public let VectorIndexRows: Int64 = 256
|
||||||
|
public let VectorIndexCols: Int64 = 256
|
||||||
|
public let VectorIndexSize: Int64 = 8
|
||||||
|
public let IPv4SegmentIndexSize: Int64 = 14 // 4 + 4 + 2 + 4
|
||||||
|
public let IPv6SegmentIndexSize: Int64 = 38 // 16 + 16 + 2 + 4
|
||||||
|
|
||||||
|
public let Structure20: Int64 = 2
|
||||||
|
public let Structure30: Int64 = 3
|
||||||
|
|
||||||
|
// xdb header parsed from the first 256 bytes of the xdb file
|
||||||
|
public class Header {
|
||||||
|
public let version: UInt16
|
||||||
|
public let indexPolicy: UInt16
|
||||||
|
public let createdAt: UInt32
|
||||||
|
public let startIndexPtr: UInt32
|
||||||
|
public let endIndexPtr: UInt32
|
||||||
|
public let ipVersion: UInt16
|
||||||
|
public let runtimePtrBytes: UInt16
|
||||||
|
|
||||||
|
public init(
|
||||||
|
ver: UInt16,
|
||||||
|
idxPolicy: UInt16,
|
||||||
|
createAt: UInt32,
|
||||||
|
sPtr: UInt32,
|
||||||
|
ePtr: UInt32,
|
||||||
|
ipVer: UInt16,
|
||||||
|
rtPtrBytes: UInt16
|
||||||
|
) {
|
||||||
|
this.version = ver
|
||||||
|
this.indexPolicy = idxPolicy
|
||||||
|
this.createdAt = createAt
|
||||||
|
this.startIndexPtr = sPtr
|
||||||
|
this.endIndexPtr = ePtr
|
||||||
|
this.ipVersion = ipVer
|
||||||
|
this.runtimePtrBytes = rtPtrBytes
|
||||||
|
}
|
||||||
|
|
||||||
|
public func toString(): String {
|
||||||
|
return "{version: ${this.version}, index_policy: ${this.indexPolicy}, " +
|
||||||
|
"created_at: ${this.createdAt}, start_index_ptr: ${this.startIndexPtr}, " +
|
||||||
|
"end_index_ptr: ${this.endIndexPtr}, ip_version: ${this.ipVersion}, " +
|
||||||
|
"runtime_ptr_bytes: ${this.runtimePtrBytes}}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read a little-endian UInt16 from byte array at offset
|
||||||
|
func readHeaderUInt16(buf: Array<Byte>, offset: Int64): UInt16 {
|
||||||
|
let b0 = UInt16(buf[offset])
|
||||||
|
let b1 = UInt16(buf[offset + 1]) << 8
|
||||||
|
return b0 | b1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read a little-endian UInt32 from byte array at offset
|
||||||
|
func readHeaderUInt32(buf: Array<Byte>, offset: Int64): UInt32 {
|
||||||
|
let b0 = UInt32(buf[offset])
|
||||||
|
let b1 = UInt32(buf[offset + 1]) << 8
|
||||||
|
let b2 = UInt32(buf[offset + 2]) << 16
|
||||||
|
let b3 = UInt32(buf[offset + 3]) << 24
|
||||||
|
return b0 | b1 | b2 | b3
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a Header from raw byte buffer (first 256 bytes of xdb file)
|
||||||
|
public func newHeaderFromBytes(buf: Array<Byte>): Header {
|
||||||
|
return Header(
|
||||||
|
readHeaderUInt16(buf, 0),
|
||||||
|
readHeaderUInt16(buf, 2),
|
||||||
|
readHeaderUInt32(buf, 4),
|
||||||
|
readHeaderUInt32(buf, 8),
|
||||||
|
readHeaderUInt32(buf, 12),
|
||||||
|
readHeaderUInt16(buf, 16),
|
||||||
|
readHeaderUInt16(buf, 18)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,146 @@
|
||||||
|
package ip2region.xdb
|
||||||
|
|
||||||
|
import std.fs.*
|
||||||
|
import std.io.SeekPosition
|
||||||
|
|
||||||
|
// Searcher for ip2region xdb database
|
||||||
|
public class Searcher {
|
||||||
|
let version: Version
|
||||||
|
let handle: File
|
||||||
|
var ioCount: Int64
|
||||||
|
let vectorIndex: Array<Byte>
|
||||||
|
let contentBuff: Array<Byte>
|
||||||
|
let mode: Int64
|
||||||
|
|
||||||
|
public let FileOnlyMode: Int64 = 0
|
||||||
|
public let VectorIndexMode: Int64 = 1
|
||||||
|
public let ContentBuffMode: Int64 = 2
|
||||||
|
|
||||||
|
public init(ver: Version, dbFile: String) {
|
||||||
|
this.version = ver
|
||||||
|
this.handle = File(dbFile, Read)
|
||||||
|
this.vectorIndex = Array<Byte>(0, repeat: 0)
|
||||||
|
this.contentBuff = Array<Byte>(0, repeat: 0)
|
||||||
|
this.mode = FileOnlyMode
|
||||||
|
this.ioCount = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
public init(ver: Version, dbFile: String, vIndex: Array<Byte>) {
|
||||||
|
this.version = ver
|
||||||
|
this.handle = File(dbFile, Read)
|
||||||
|
this.vectorIndex = vIndex
|
||||||
|
this.contentBuff = Array<Byte>(0, repeat: 0)
|
||||||
|
this.mode = VectorIndexMode
|
||||||
|
this.ioCount = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
public init(ver: Version, cBuff: Array<Byte>, dbPath: String) {
|
||||||
|
this.version = ver
|
||||||
|
this.handle = File(dbPath, Read)
|
||||||
|
this.vectorIndex = Array<Byte>(0, repeat: 0)
|
||||||
|
this.contentBuff = cBuff
|
||||||
|
this.mode = ContentBuffMode
|
||||||
|
this.ioCount = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
public func close() {
|
||||||
|
this.handle.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
public func getIOCount(): Int64 {
|
||||||
|
return this.ioCount
|
||||||
|
}
|
||||||
|
|
||||||
|
public func searchByString(ipStr: String): String {
|
||||||
|
let ipBytes = parseIP(ipStr)
|
||||||
|
return this.search(ipBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
public func search(ip: Array<Byte>): String {
|
||||||
|
this.ioCount = 0
|
||||||
|
|
||||||
|
let il0 = Int64(ip[0])
|
||||||
|
let il1 = Int64(ip[1])
|
||||||
|
let idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize
|
||||||
|
|
||||||
|
var sPtr: Int64 = 0
|
||||||
|
var ePtr: Int64 = 0
|
||||||
|
|
||||||
|
if (this.mode == VectorIndexMode) {
|
||||||
|
sPtr = readLEUint32(this.vectorIndex, idx)
|
||||||
|
ePtr = readLEUint32(this.vectorIndex, idx + 4)
|
||||||
|
} else if (this.mode == ContentBuffMode) {
|
||||||
|
sPtr = readLEUint32(this.contentBuff, HeaderInfoLength + idx)
|
||||||
|
ePtr = readLEUint32(this.contentBuff, HeaderInfoLength + idx + 4)
|
||||||
|
} else {
|
||||||
|
let buff = Array<Byte>(VectorIndexSize, repeat: 0)
|
||||||
|
this.readFromFile(HeaderInfoLength + idx, buff)
|
||||||
|
sPtr = readLEUint32(buff, 0)
|
||||||
|
ePtr = readLEUint32(buff, 4)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sPtr == 0 || ePtr == 0) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
let segIndexSize = this.version.segmentIndexSize
|
||||||
|
let bytes = this.version.bytes
|
||||||
|
let dBytes = bytes * 2
|
||||||
|
var dataLen: Int64 = 0
|
||||||
|
var dataPtr: Int64 = 0
|
||||||
|
|
||||||
|
var l: Int64 = 0
|
||||||
|
var h: Int64 = (ePtr - sPtr) / segIndexSize
|
||||||
|
let buff = Array<Byte>(segIndexSize, repeat: 0)
|
||||||
|
|
||||||
|
while (l <= h) {
|
||||||
|
let m = (l + h) / 2
|
||||||
|
let p = sPtr + m * segIndexSize
|
||||||
|
|
||||||
|
this.read(p, buff)
|
||||||
|
this.ioCount = this.ioCount + 1
|
||||||
|
|
||||||
|
if (this.compareIP(ip, buff, 0) < 0) {
|
||||||
|
h = m - 1
|
||||||
|
} else if (this.compareIP(ip, buff, bytes) > 0) {
|
||||||
|
l = m + 1
|
||||||
|
} else {
|
||||||
|
dataLen = readLEUint16(buff, dBytes)
|
||||||
|
dataPtr = readLEUint32(buff, dBytes + 2)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataLen == 0) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
let regionBuff = Array<Byte>(dataLen, repeat: 0)
|
||||||
|
this.read(dataPtr, regionBuff)
|
||||||
|
return String.fromUtf8(regionBuff)
|
||||||
|
}
|
||||||
|
|
||||||
|
func read(offset: Int64, buff: Array<Byte>) {
|
||||||
|
if (this.mode == ContentBuffMode) {
|
||||||
|
for (i in 0..buff.size) {
|
||||||
|
buff[i] = this.contentBuff[offset + i]
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.readFromFile(offset, buff)
|
||||||
|
()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFromFile(offset: Int64, buff: Array<Byte>) {
|
||||||
|
this.handle.seek(SeekPosition.Begin(offset))
|
||||||
|
this.handle.read(buff)
|
||||||
|
()
|
||||||
|
}
|
||||||
|
|
||||||
|
func compareIP(ip: Array<Byte>, entry: Array<Byte>, offset: Int64): Int64 {
|
||||||
|
if (this.version.id == IPv4VersionNo) {
|
||||||
|
return compareIPv4(ip, entry, offset)
|
||||||
|
}
|
||||||
|
return compareIPv6(ip, entry, offset)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
package ip2region.xdb
|
||||||
|
|
||||||
|
import std.net.IPAddress
|
||||||
|
|
||||||
|
// --- IP address parsing and formatting
|
||||||
|
|
||||||
|
// Parse IP string to byte array
|
||||||
|
// For IPv4, returns 4 bytes in big-endian order (network byte order)
|
||||||
|
// For IPv6, returns 16 bytes in big-endian order
|
||||||
|
public func parseIP(ip: String): Array<Byte> {
|
||||||
|
let addr = IPAddress.parse(ip)
|
||||||
|
return addr.getAddressBytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert IP byte array back to string
|
||||||
|
public func ipToString(ip: Array<Byte>): String {
|
||||||
|
if (ip.size == 4) {
|
||||||
|
return "${Int64(ip[0])}.${Int64(ip[1])}.${Int64(ip[2])}.${Int64(ip[3])}"
|
||||||
|
}
|
||||||
|
// IPv6 simplified format
|
||||||
|
var result = ""
|
||||||
|
for (i in 0..ip.size) {
|
||||||
|
if (i > 0 && i % 2 == 0) {
|
||||||
|
result = result + ":"
|
||||||
|
}
|
||||||
|
result = result + "${Int64(ip[i])}"
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare two IP byte arrays.
|
||||||
|
// Returns: -1 if ip1 < ip2, 0 if equal, 1 if ip1 > ip2
|
||||||
|
public func ipCompare(ip1: Array<Byte>, ip2: Array<Byte>): Int64 {
|
||||||
|
var len = ip1.size
|
||||||
|
if (ip2.size < len) {
|
||||||
|
len = ip2.size
|
||||||
|
}
|
||||||
|
for (i in 0..len) {
|
||||||
|
if (ip1[i] < ip2[i]) {
|
||||||
|
return -1
|
||||||
|
} else if (ip1[i] > ip2[i]) {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ip1.size < ip2.size) {
|
||||||
|
return -1
|
||||||
|
} else if (ip1.size > ip2.size) {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Binary data reading helpers for xdb format
|
||||||
|
|
||||||
|
// Read a little-endian UInt32 from a byte buffer at the given offset
|
||||||
|
public func readLEUint32(buf: Array<Byte>, offset: Int64): Int64 {
|
||||||
|
let b0 = Int64(buf[offset])
|
||||||
|
let b1 = Int64(buf[offset + 1]) << 8
|
||||||
|
let b2 = Int64(buf[offset + 2]) << 16
|
||||||
|
let b3 = Int64(buf[offset + 3]) << 24
|
||||||
|
return b0 | b1 | b2 | b3
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read a little-endian UInt16 from a byte buffer at the given offset
|
||||||
|
public func readLEUint16(buf: Array<Byte>, offset: Int64): Int64 {
|
||||||
|
let b0 = Int64(buf[offset])
|
||||||
|
let b1 = Int64(buf[offset + 1]) << 8
|
||||||
|
return b0 | b1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare IPv4 address (big-endian) with xdb stored IPv4 (little-endian)
|
||||||
|
// xdb stores IPv4 in little-endian, so we compare ip[i] with xdbEntry[3-i]
|
||||||
|
public func compareIPv4(ip: Array<Byte>, xdbEntry: Array<Byte>, entryOffset: Int64): Int64 {
|
||||||
|
for (i in 0..4) {
|
||||||
|
let ipByte = Int64(ip[i])
|
||||||
|
let xdbByte = Int64(xdbEntry[entryOffset + 3 - i])
|
||||||
|
if (ipByte < xdbByte) {
|
||||||
|
return -1
|
||||||
|
} else if (ipByte > xdbByte) {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load the vector index from a content buffer
|
||||||
|
// The vector index starts at HeaderInfoLength and has VectorIndexRows * VectorIndexCols * VectorIndexSize bytes
|
||||||
|
public func loadVectorIndex(cBuff: Array<Byte>): Array<Byte> {
|
||||||
|
let start = HeaderInfoLength
|
||||||
|
let len = VectorIndexRows * VectorIndexCols * VectorIndexSize
|
||||||
|
let vi = Array<Byte>(len, repeat: 0)
|
||||||
|
for (i in 0..len) {
|
||||||
|
vi[i] = cBuff[start + i]
|
||||||
|
}
|
||||||
|
return vi
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare IPv6 address (big-endian) with xdb stored IPv6 (big-endian)
|
||||||
|
public func compareIPv6(ip: Array<Byte>, xdbEntry: Array<Byte>, entryOffset: Int64): Int64 {
|
||||||
|
for (i in 0..16) {
|
||||||
|
if (ip[i] < xdbEntry[entryOffset + i]) {
|
||||||
|
return -1
|
||||||
|
} else if (ip[i] > xdbEntry[entryOffset + i]) {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
package ip2region.xdb
|
||||||
|
|
||||||
|
// IP version definitions for ip2region xdb searcher
|
||||||
|
public class Version {
|
||||||
|
public let id: Int64
|
||||||
|
public let name: String
|
||||||
|
public let bytes: Int64
|
||||||
|
public let segmentIndexSize: Int64
|
||||||
|
|
||||||
|
public init(id: Int64, name: String, bytes: Int64, segmentIndexSize: Int64) {
|
||||||
|
this.id = id
|
||||||
|
this.name = name
|
||||||
|
this.bytes = bytes
|
||||||
|
this.segmentIndexSize = segmentIndexSize
|
||||||
|
}
|
||||||
|
|
||||||
|
public func toString(): String {
|
||||||
|
return "{id: ${this.id}, name: ${this.name}, bytes: ${this.bytes}, " +
|
||||||
|
"segment_index_size: ${this.segmentIndexSize}}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public let IPv4VersionNo: Int64 = 4
|
||||||
|
public let IPv6VersionNo: Int64 = 6
|
||||||
|
|
||||||
|
// Pre-defined IP versions
|
||||||
|
public let IPv4 = Version(4, "IPv4", 4, 14)
|
||||||
|
public let IPv6 = Version(6, "IPv6", 16, 38)
|
||||||
|
|
||||||
|
// Determine IP version from header info
|
||||||
|
public func versionFromHeader(header: Header): Version {
|
||||||
|
// Old structure (2.0) with IPv4 only
|
||||||
|
if (Int64(header.version) == Structure20) {
|
||||||
|
return IPv4
|
||||||
|
}
|
||||||
|
// Structure 3.0+
|
||||||
|
if (Int64(header.ipVersion) == IPv4VersionNo) {
|
||||||
|
return IPv4
|
||||||
|
}
|
||||||
|
return IPv6
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
bin/
|
||||||
|
|
@ -0,0 +1,137 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region C++ query client
|
||||||
|
|
||||||
|
## 0. File Description
|
||||||
|
|
||||||
|
```
|
||||||
|
Makefile --------- Build
|
||||||
|
|
||||||
|
src ------------------ Source directory
|
||||||
|
src/base.* ----------- Constants and utility functions
|
||||||
|
src/ip.* ------------- IP processing implementation
|
||||||
|
src/header.* --------- xdb header parsing implementation
|
||||||
|
src/search.* --------- xdb search implementation
|
||||||
|
src/bench.* ---------- Search benchmarking implementation
|
||||||
|
src/make.* ----------- xdb file generation implementation
|
||||||
|
src/edit.* ----------- Raw data editing implementation
|
||||||
|
|
||||||
|
test ---------------- Test directory
|
||||||
|
test/header.cc ------ Test header
|
||||||
|
test/search.cc ------ Test search
|
||||||
|
test/bench.cc ------- Benchmarking
|
||||||
|
test/make.cc -------- Generate xdb file
|
||||||
|
test/edit_v4.cc ----- Test raw data editing (ipv4)
|
||||||
|
test/edit_v6.cc ----- Test raw data editing (ipv6)
|
||||||
|
|
||||||
|
|
||||||
|
bin --------------- Executable directory (generated via make)
|
||||||
|
bin/header -------- Test header
|
||||||
|
bin/search -------- Test search
|
||||||
|
bin/bench --------- Benchmarking
|
||||||
|
bin/make ---------- Generate xdb file
|
||||||
|
bin/edit_v4 ------- Test raw data editing (ipv4)
|
||||||
|
bin/edit_v6 ------- Test raw data editing (ipv6)
|
||||||
|
|
||||||
|
readme.md --------- readme
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
## 1. Compilation
|
||||||
|
|
||||||
|
```
|
||||||
|
$ make
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. Search
|
||||||
|
|
||||||
|
### 2.1 Example
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
#include "src/search.h"
|
||||||
|
|
||||||
|
// IP Version: xdb::ipv4 xdb::ipv6
|
||||||
|
// Policy: xdb::policy_file xdb::policy_vector xdb::policy_content
|
||||||
|
// No cache Partial cache Full cache
|
||||||
|
int main() {
|
||||||
|
std::string xdb_name = "../../data/ip2region_v6.xdb";
|
||||||
|
int version = xdb::ipv6;
|
||||||
|
int policy = xdb::policy_content;
|
||||||
|
std::string ip = "2001:200:124::";
|
||||||
|
|
||||||
|
xdb::search_t s(xdb_name, version, policy);
|
||||||
|
std::cout << s.search(ip) << std::endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// $ g++ src/*.cc 1.cc --- Compile
|
||||||
|
// $ ./a.out ------------- Test
|
||||||
|
// Japan|Tokyo|Asagaya-minami|WIDE Project|JP
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 Test xdb Header
|
||||||
|
|
||||||
|
```
|
||||||
|
$ ./bin/header
|
||||||
|
Test IPv4
|
||||||
|
Version: 3
|
||||||
|
Cache Policy: 1
|
||||||
|
File Generation Time: 2025-09-06 02:24:16
|
||||||
|
Index Start Address: 955933
|
||||||
|
Index End Address: 11042415
|
||||||
|
IP Version: 4
|
||||||
|
Pointer Bytes: 4
|
||||||
|
|
||||||
|
Test IPv6
|
||||||
|
Version: 3
|
||||||
|
Cache Policy: 1
|
||||||
|
File Generation Time: 2025-10-17 04:41:04
|
||||||
|
Index Start Address: 3094259
|
||||||
|
Index End Address: 36258303
|
||||||
|
IP Version: 6
|
||||||
|
Pointer Bytes: 4
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 Test Search
|
||||||
|
|
||||||
|
```
|
||||||
|
$ ./bin/search
|
||||||
|
Test IPv4 No cache: Success
|
||||||
|
Test IPv4 Partial cache: Success
|
||||||
|
Test IPv4 Full cache: Success
|
||||||
|
Test IPv6 No cache: Success
|
||||||
|
Test IPv6 Partial cache: Success
|
||||||
|
Test IPv6 Full cache: Success
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. Benchmarking and Correctness Verification
|
||||||
|
|
||||||
|
```
|
||||||
|
./bin/bench
|
||||||
|
Test IPv4, No cache, total: 3910284, took: 27.60s, cost: 6.59μs/op, io count: 28227147
|
||||||
|
Test IPv4, Partial cache, total: 3910284, took: 21.85s, cost: 5.15μs/op, io count: 24316863
|
||||||
|
Test IPv4, Full cache, total: 3910284, took: 2.26s, cost: 0.25μs/op, io count: 0
|
||||||
|
Test IPv6, No cache, total: 4792520, took: 100.40s, cost: 20.22μs/op, io count: 80758866
|
||||||
|
Test IPv6, Partial cache, total: 4792520, took: 93.06s, cost: 18.71μs/op, io count: 75966346
|
||||||
|
Test IPv6, Full cache, total: 4792520, took: 6.24s, cost: 0.81μs/op, io count: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. Generate xdb File
|
||||||
|
|
||||||
|
### 4.1 Generate xdb File
|
||||||
|
|
||||||
|
```
|
||||||
|
$ ./bin/make
|
||||||
|
Generate ipv4 xdb file, took: 0.57s
|
||||||
|
Generate ipv6 xdb file, took: 1.24s
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Raw Data Editing
|
||||||
|
|
||||||
|
### 5.1. Instructions for Use
|
||||||
|
|
||||||
|
* New IP attribution files can contain empty lines
|
||||||
|
* New IP attribution files can be out of order; the program will automatically sort them
|
||||||
|
* New IP attribution files can overlap; as long as there is no ambiguity, the program will automatically merge them
|
||||||
|
* The final result will automatically merge adjacent lines with the same attribution
|
||||||
|
* For the following tests, the original file uses the data file provided in the repository, and the new file uses 1.txt in the current directory
|
||||||
|
|
@ -0,0 +1,125 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region C++ 查询客户端
|
||||||
|
|
||||||
|
## 0. 文件说明
|
||||||
|
```
|
||||||
|
Makefile --------- 构建
|
||||||
|
|
||||||
|
src ------------------ 源文件目录
|
||||||
|
src/base.* ----------- 常量及工具函数
|
||||||
|
src/ip.* ------------- 实现 IP 处理
|
||||||
|
src/header.* --------- 实现 xdb 头部解析
|
||||||
|
src/search.* --------- 实现 xdb 查找
|
||||||
|
src/bench.* ---------- 实现 查找 测速
|
||||||
|
src/make.* ----------- 实现 生成 xdb 文件
|
||||||
|
src/edit.* ----------- 实现 原始数据编辑
|
||||||
|
|
||||||
|
test ---------------- 测试目录
|
||||||
|
test/header.cc ------ 测试 头部
|
||||||
|
test/search.cc ------ 测试 查找
|
||||||
|
test/bench.cc ------- 测速
|
||||||
|
test/make.cc -------- 生成 xdb 文件
|
||||||
|
test/edit_v4.cc ----- 测试 原始数据编辑(ipv4)
|
||||||
|
test/edit_v6.cc ----- 测试 原始数据编辑(ipv6)
|
||||||
|
|
||||||
|
|
||||||
|
bin --------------- 可执行文件目录(通过 make 生成)
|
||||||
|
bin/header -------- 测试 头部
|
||||||
|
bin/search -------- 测试 查找
|
||||||
|
bin/bench --------- 测速
|
||||||
|
bin/make ---------- 生成 xdb 文件
|
||||||
|
bin/edit_v4 ------- 测试 原始数据编辑(ipv4)
|
||||||
|
bin/edit_v6 ------- 测试 原始数据编辑(ipv6)
|
||||||
|
|
||||||
|
readme.md --------- readme
|
||||||
|
```
|
||||||
|
|
||||||
|
## 1. 编译
|
||||||
|
```
|
||||||
|
$ make
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. 查找
|
||||||
|
### 2.1 示例
|
||||||
|
```cpp
|
||||||
|
#include "src/search.h"
|
||||||
|
|
||||||
|
// IP 版本: xdb::ipv4 xdb::ipv6
|
||||||
|
// 策略: xdb::policy_file xdb::policy_vector xdb::policy_content
|
||||||
|
// 不缓存 部分缓存 全部缓存
|
||||||
|
int main() {
|
||||||
|
std::string xdb_name = "../../data/ip2region_v6.xdb";
|
||||||
|
int version = xdb::ipv6;
|
||||||
|
int policy = xdb::policy_content;
|
||||||
|
std::string ip = "2001:200:124::";
|
||||||
|
|
||||||
|
xdb::search_t s(xdb_name, version, policy);
|
||||||
|
std::cout << s.search(ip) << std::endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// $ g++ src/*.cc 1.cc --- 编译
|
||||||
|
// $ ./a.out ------------- 测试
|
||||||
|
// Japan|Tokyo|Asagaya-minami|WIDE Project|JP
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 测试 xdb 头部
|
||||||
|
```
|
||||||
|
$ ./bin/header
|
||||||
|
测试 IPv4
|
||||||
|
版本号: 3
|
||||||
|
缓存策略: 1
|
||||||
|
文件生成时间: 2025-09-06 02:24:16
|
||||||
|
索引起始地址: 955933
|
||||||
|
索引结束地址: 11042415
|
||||||
|
IP版本: 4
|
||||||
|
指针字节数: 4
|
||||||
|
|
||||||
|
测试 IPv6
|
||||||
|
版本号: 3
|
||||||
|
缓存策略: 1
|
||||||
|
文件生成时间: 2025-10-17 04:41:04
|
||||||
|
索引起始地址: 3094259
|
||||||
|
索引结束地址: 36258303
|
||||||
|
IP版本: 6
|
||||||
|
指针字节数: 4
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 测试查找
|
||||||
|
```
|
||||||
|
$ ./bin/search
|
||||||
|
测试 IPv4 不缓存: 成功
|
||||||
|
测试 IPv4 部分缓存: 成功
|
||||||
|
测试 IPv4 全部缓存: 成功
|
||||||
|
测试 IPv6 不缓存: 成功
|
||||||
|
测试 IPv6 部分缓存: 成功
|
||||||
|
测试 IPv6 全部缓存: 成功
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. 测速以及检验正确性
|
||||||
|
```
|
||||||
|
./bin/bench
|
||||||
|
测试 IPv4, 不缓存, total: 3910284, took: 27.60s, cost: 6.59μs/op, io count: 28227147
|
||||||
|
测试 IPv4, 部分缓存, total: 3910284, took: 21.85s, cost: 5.15μs/op, io count: 24316863
|
||||||
|
测试 IPv4, 全部缓存, total: 3910284, took: 2.26s, cost: 0.25μs/op, io count: 0
|
||||||
|
测试 IPv6, 不缓存, total: 4792520, took: 100.40s, cost: 20.22μs/op, io count: 80758866
|
||||||
|
测试 IPv6, 部分缓存, total: 4792520, took: 93.06s, cost: 18.71μs/op, io count: 75966346
|
||||||
|
测试 IPv6, 全部缓存, total: 4792520, took: 6.24s, cost: 0.81μs/op, io count: 0
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 生成 xdb 文件
|
||||||
|
### 4.1 生成 xdb 文件
|
||||||
|
```
|
||||||
|
$ ./bin/make
|
||||||
|
生成 ipv4 的 xdb 文件, took: 0.57s
|
||||||
|
生成 ipv6 的 xdb 文件, took: 1.24s
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. 原始数据编辑
|
||||||
|
### 5.1. 使用说明
|
||||||
|
* 新的IP归属地文件可以包含空行
|
||||||
|
* 新的IP归属地文件顺序可以乱序, 程序会自动排序
|
||||||
|
* 新的IP归属地文件顺序可以重叠, 只要无二义性, 程序会自动合并
|
||||||
|
* 最终的结果会将相邻的且归属地相同的行自动合并
|
||||||
|
* 以下测试, 原文件使用仓库自带的数据文件, 新文件使用当前目录下的 1.txt
|
||||||
|
|
@ -1,563 +0,0 @@
|
||||||
# ip2region xdb C++ 实现
|
|
||||||
|
|
||||||
## 0. 文件说明
|
|
||||||
```
|
|
||||||
Makefile --------- 构建
|
|
||||||
|
|
||||||
src ------------------ 源文件目录
|
|
||||||
src/base.* ----------- 常量及工具函数
|
|
||||||
src/ip.* ------------- 实现 IP 处理
|
|
||||||
src/header.* --------- 实现 xdb 头部解析
|
|
||||||
src/search.* --------- 实现 xdb 查找
|
|
||||||
src/bench.* ---------- 实现 查找 测速
|
|
||||||
src/make.* ----------- 实现 生成 xdb 文件
|
|
||||||
src/edit.* ----------- 实现 原始数据编辑
|
|
||||||
|
|
||||||
test ---------------- 测试目录
|
|
||||||
test/header.cc ------ 测试 头部
|
|
||||||
test/search.cc ------ 测试 查找
|
|
||||||
test/bench.cc ------- 测速
|
|
||||||
test/make.cc -------- 生成 xdb 文件
|
|
||||||
test/edit_v4.cc ----- 测试 原始数据编辑(ipv4)
|
|
||||||
test/edit_v6.cc ----- 测试 原始数据编辑(ipv6)
|
|
||||||
|
|
||||||
|
|
||||||
bin --------------- 可执行文件目录(通过 make 生成)
|
|
||||||
bin/header -------- 测试 头部
|
|
||||||
bin/search -------- 测试 查找
|
|
||||||
bin/bench --------- 测速
|
|
||||||
bin/make ---------- 生成 xdb 文件
|
|
||||||
bin/edit_v4 ------- 测试 原始数据编辑(ipv4)
|
|
||||||
bin/edit_v6 ------- 测试 原始数据编辑(ipv6)
|
|
||||||
|
|
||||||
readme.md --------- readme
|
|
||||||
```
|
|
||||||
|
|
||||||
## 1. 编译
|
|
||||||
```
|
|
||||||
$ make
|
|
||||||
```
|
|
||||||
|
|
||||||
## 2. 查找
|
|
||||||
### 2.1 示例
|
|
||||||
```
|
|
||||||
#include "src/search.h"
|
|
||||||
|
|
||||||
// IP 版本: xdb::ipv4 xdb::ipv6
|
|
||||||
// 策略: xdb::policy_file xdb::policy_vector xdb::policy_content
|
|
||||||
// 不缓存 部分缓存 全部缓存
|
|
||||||
int main() {
|
|
||||||
std::string xdb_name = "../../data/ip2region_v6.xdb";
|
|
||||||
int version = xdb::ipv6;
|
|
||||||
int policy = xdb::policy_content;
|
|
||||||
std::string ip = "2001:200:124::";
|
|
||||||
|
|
||||||
xdb::search_t s(xdb_name, version, policy);
|
|
||||||
std::cout << s.search(ip) << std::endl;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// $ g++ src/*.cc 1.cc --- 编译
|
|
||||||
// $ ./a.out ------------- 测试
|
|
||||||
// 日本|东京都|千代田区|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2.2 测试 xdb 头部
|
|
||||||
```
|
|
||||||
$ ./bin/header
|
|
||||||
测试 IPv4
|
|
||||||
版本号: 3
|
|
||||||
缓存策略: 1
|
|
||||||
文件生成时间: 2025-09-06 02:24:16
|
|
||||||
索引起始地址: 955933
|
|
||||||
索引结束地址: 11042415
|
|
||||||
IP版本: 4
|
|
||||||
指针字节数: 4
|
|
||||||
|
|
||||||
测试 IPv6
|
|
||||||
版本号: 3
|
|
||||||
缓存策略: 1
|
|
||||||
文件生成时间: 2025-10-17 04:41:04
|
|
||||||
索引起始地址: 3094259
|
|
||||||
索引结束地址: 36258303
|
|
||||||
IP版本: 6
|
|
||||||
指针字节数: 4
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2.3 测试查找
|
|
||||||
```
|
|
||||||
$ ./bin/search
|
|
||||||
测试 IPv4 不缓存: 成功
|
|
||||||
测试 IPv4 部分缓存: 成功
|
|
||||||
测试 IPv4 全部缓存: 成功
|
|
||||||
测试 IPv6 不缓存: 成功
|
|
||||||
测试 IPv6 部分缓存: 成功
|
|
||||||
测试 IPv6 全部缓存: 成功
|
|
||||||
```
|
|
||||||
|
|
||||||
## 3. 测速以及检验正确性
|
|
||||||
```
|
|
||||||
./bin/bench
|
|
||||||
测试 IPv4, 不缓存, total: 3910284, took: 27.60s, cost: 6.59μs/op, io count: 28227147
|
|
||||||
测试 IPv4, 部分缓存, total: 3910284, took: 21.85s, cost: 5.15μs/op, io count: 24316863
|
|
||||||
测试 IPv4, 全部缓存, total: 3910284, took: 2.26s, cost: 0.25μs/op, io count: 0
|
|
||||||
测试 IPv6, 不缓存, total: 4792520, took: 100.40s, cost: 20.22μs/op, io count: 80758866
|
|
||||||
测试 IPv6, 部分缓存, total: 4792520, took: 93.06s, cost: 18.71μs/op, io count: 75966346
|
|
||||||
测试 IPv6, 全部缓存, total: 4792520, took: 6.24s, cost: 0.81μs/op, io count: 0
|
|
||||||
```
|
|
||||||
|
|
||||||
## 4. 生成 xdb 文件
|
|
||||||
### 4.1 生成 xdb 文件
|
|
||||||
```
|
|
||||||
$ ./bin/make
|
|
||||||
生成 ipv4 的 xdb 文件, took: 0.57s
|
|
||||||
生成 ipv6 的 xdb 文件, took: 1.24s
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4.2 测试正确性
|
|
||||||
```
|
|
||||||
# ipv4 --- 只有时间不同
|
|
||||||
$ diff <(xxd ./ip2region_v4.xdb) <(xxd ../../data/ip2region_v4.xdb)
|
|
||||||
1c1
|
|
||||||
< 00000000: 0300 0100 9f2f 2969 1d96 0e00 6f7e a800 ...../)i....o~..
|
|
||||||
---
|
|
||||||
> 00000000: 0300 0100 509b bb68 1d96 0e00 6f7e a800 ....P..h....o~..
|
|
||||||
|
|
||||||
# ipv6 --- 只有时间不同
|
|
||||||
$ diff <(xxd ./ip2region_v6.xdb) <(xxd ../../data/ip2region_v6.xdb)
|
|
||||||
1c1
|
|
||||||
< 00000000: 0300 0100 a02f 2969 f336 2f00 ff41 2902 ...../)i.6/..A).
|
|
||||||
---
|
|
||||||
> 00000000: 0300 0100 e0c8 f168 f336 2f00 ff41 2902 .......h.6/..A).
|
|
||||||
```
|
|
||||||
|
|
||||||
## 5. 原始数据编辑
|
|
||||||
### 5.1. 使用说明
|
|
||||||
* 新的IP归属地文件可以包含空行
|
|
||||||
* 新的IP归属地文件顺序可以乱序, 程序会自动排序
|
|
||||||
* 新的IP归属地文件顺序可以重叠, 只要无二义性, 程序会自动合并
|
|
||||||
* 最终的结果会将相邻的且归属地相同的行自动合并
|
|
||||||
* 以下测试, 原文件使用仓库自带的数据文件, 新文件使用当前目录下的 1.txt
|
|
||||||
|
|
||||||
### 5.2. 数据正确性测试 -- ipv4
|
|
||||||
#### 测试一: 测试数据文件包含空行以及重复的情况
|
|
||||||
```
|
|
||||||
$ cat -n 1.txt
|
|
||||||
1
|
|
||||||
2 1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
3
|
|
||||||
4 1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
5
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.80s
|
|
||||||
$ git diff ../../data/
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..5d1fdfa 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -7,7 +7,7 @@
|
|
||||||
1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
-1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
+1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试二: 测试数据文件乱序以及数据有交叉, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.128.5|1.0.128.255|测试归属地
|
|
||||||
1.0.128.0|1.0.128.9|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.88s
|
|
||||||
$ git diff ../../data/
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..5d1fdfa 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -7,7 +7,7 @@
|
|
||||||
1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
-1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
+1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试三: 测试数据文件重叠, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.128.0|1.0.128.8|测试归属地
|
|
||||||
1.0.128.7|1.0.128.255|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.91s
|
|
||||||
$ git diff ../../data/
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..5d1fdfa 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -7,7 +7,7 @@
|
|
||||||
1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
-1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
+1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试四: 测试数据文件重叠, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.128.0|1.0.128.8|测试归属地
|
|
||||||
1.0.128.8|1.0.128.255|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.81s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..5d1fdfa 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -7,7 +7,7 @@
|
|
||||||
1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
-1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
+1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试五: 测试数据文件连接, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.128.0|1.0.128.8|测试归属地
|
|
||||||
1.0.128.9|1.0.128.255|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.71s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..5d1fdfa 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -7,7 +7,7 @@
|
|
||||||
1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
-1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
+1.0.128.0|1.0.128.255|测试归属地
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试六: 测试数据文件重叠, 归属地不同情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.128.0|1.0.128.8|测试归属地123
|
|
||||||
1.0.128.8|1.0.128.255|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
数据有二义性: 1.0.128.0|1.0.128.8|测试归属地123, 1.0.128.8|1.0.128.255|测试归属地
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试七: 测试数据文件连接, 归属地不同情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.128.0|1.0.128.8|测试归属地123
|
|
||||||
1.0.128.9|1.0.128.255|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.75s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..976e6bf 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -7,7 +7,8 @@
|
|
||||||
1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
-1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
+1.0.128.0|1.0.128.8|测试归属地123
|
|
||||||
+1.0.128.9|1.0.128.255|测试归属地
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
1.0.133.0|1.0.133.255|泰国|素攀武里府|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试八: 测试将一个IP数据拆成多个IP
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
36.136.1.0|36.136.7.255|中国|0|广西|来宾市|移动
|
|
||||||
36.136.8.0|36.136.15.255|中国|0|广西|玉林市|移动
|
|
||||||
36.136.16.0|36.136.23.255|中国|0|广西|河池市|移动
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.80s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..f895c2f 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -54778,7 +54778,11 @@
|
|
||||||
36.134.84.0|36.134.85.255|中国|安徽省|合肥市|移动
|
|
||||||
36.134.86.0|36.134.87.255|中国|广西|南宁市|移动
|
|
||||||
36.134.88.0|36.134.89.255|中国|内蒙古|呼和浩特市|移动
|
|
||||||
-36.134.90.0|36.141.255.255|中国|0|0|移动
|
|
||||||
+36.134.90.0|36.136.0.255|中国|0|0|移动
|
|
||||||
+36.136.1.0|36.136.7.255|中国|0|广西|来宾市|移动
|
|
||||||
+36.136.8.0|36.136.15.255|中国|0|广西|玉林市|移动
|
|
||||||
+36.136.16.0|36.136.23.255|中国|0|广西|河池市|移动
|
|
||||||
+36.136.24.0|36.141.255.255|中国|0|0|移动
|
|
||||||
36.142.0.0|36.142.1.255|中国|四川省|成都市|移动
|
|
||||||
36.142.2.0|36.142.31.255|中国|甘肃省|兰州市|移动
|
|
||||||
36.142.32.0|36.142.127.255|中国|甘肃省|0|移动
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试九: 测试将多个IP数据并成一个IP数据
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
1.0.16.0|1.0.127.255|测试归属地
|
|
||||||
$ ./bin/edit_v4
|
|
||||||
took: 0.76s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv4_source.txt b/data/ipv4_source.txt
|
|
||||||
index 00dacc3..756354c 100644
|
|
||||||
--- a/data/ipv4_source.txt
|
|
||||||
+++ b/data/ipv4_source.txt
|
|
||||||
@@ -3,10 +3,7 @@
|
|
||||||
1.0.1.0|1.0.3.255|中国|福建省|福州市|电信
|
|
||||||
1.0.4.0|1.0.7.255|澳大利亚|维多利亚|墨尔本|0
|
|
||||||
1.0.8.0|1.0.15.255|中国|广东省|广州市|电信
|
|
||||||
-1.0.16.0|1.0.31.255|日本|0|0|0
|
|
||||||
-1.0.32.0|1.0.63.255|中国|广东省|广州市|电信
|
|
||||||
-1.0.64.0|1.0.79.255|日本|广岛县|0|0
|
|
||||||
-1.0.80.0|1.0.127.255|日本|冈山县|0|0
|
|
||||||
+1.0.16.0|1.0.127.255|测试归属地
|
|
||||||
1.0.128.0|1.0.128.255|泰国|清莱府|0|TOT
|
|
||||||
1.0.129.0|1.0.132.191|泰国|曼谷|曼谷|TOT
|
|
||||||
1.0.132.192|1.0.132.255|泰国|Nakhon-Ratchasima|0|TOT
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5.3 数据正确性测试 -- ipv6
|
|
||||||
#### 测试一: 测试数据文件包含空行以及重复的情况
|
|
||||||
```
|
|
||||||
$ cat -n 1.txt
|
|
||||||
1
|
|
||||||
2 2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
3
|
|
||||||
4 2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
5
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.74s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..29617c4 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -2,7 +2,7 @@
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
+2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试二: 测试数据文件乱序以及数据有交叉, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:121::|2001:200:125:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.68s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..9e83b03 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -2,9 +2,8 @@
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
+2001:200:120::|2001:200:125:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
+2001:200:126::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:12b::|2001:200:130:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:131::|2001:200:132:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试三: 测试数据文件重叠, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:120::|2001:200:125:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
2001:200:125::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.75s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..7a23ba2 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -2,9 +2,8 @@
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
+2001:200:120::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
+2001:200:127::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:12b::|2001:200:130:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:131::|2001:200:132:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试四: 测试数据文件重叠, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:120::|2001:200:125::|测试归属地
|
|
||||||
2001:200:125::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.46s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..7a23ba2 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -2,9 +2,8 @@
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
+2001:200:120::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
+2001:200:127::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:12b::|2001:200:130:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:131::|2001:200:132:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试五: 测试数据文件连接, 归属地相同的情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:120::|2001:200:125:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
2001:200:126::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.79s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..7a23ba2 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -2,9 +2,8 @@
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
+2001:200:120::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
+2001:200:127::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:12b::|2001:200:130:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:131::|2001:200:132:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试六: 测试数据文件重叠, 归属地不同情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:120::|2001:200:126::|测试归属地123
|
|
||||||
2001:200:126::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
数据有二义性: 2001:200:120::|2001:200:126::|测试归属地123, 2001:200:126::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试七: 测试数据文件连接, 归属地不同情况
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:120::|2001:200:125:ffff:ffff:ffff:ffff:ffff|测试归属地123
|
|
||||||
2001:200:126::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.78s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..142f7cc 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -2,9 +2,9 @@
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
+2001:200:120::|2001:200:125:ffff:ffff:ffff:ffff:ffff|测试归属地123
|
|
||||||
+2001:200:126::|2001:200:126:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
+2001:200:127::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:12b::|2001:200:130:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:131::|2001:200:132:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试八: 测试将一个IP数据拆成多个IP
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ff11|测试归属地1
|
|
||||||
2001:200:11f:ffff:ffff:ffff:ffff:ff12|2001:200:11f:ffff:ffff:ffff:ffff:ff33|测试归属地2
|
|
||||||
2001:200:11f:ffff:ffff:ffff:ffff:ff34|2001:200:11f:ffff:ffff:ffff:ffff:ffff|测试归属地3
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.52s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..e450e27 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -1,7 +1,9 @@
|
|
||||||
1:1::|2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff|0|0|内网IP|内网IP
|
|
||||||
2001:200::|2001:200:101:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
+2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ff11|测试归属地1
|
|
||||||
+2001:200:11f:ffff:ffff:ffff:ffff:ff12|2001:200:11f:ffff:ffff:ffff:ffff:ff33|测试归属地2
|
|
||||||
+2001:200:11f:ffff:ffff:ffff:ffff:ff34|2001:200:11f:ffff:ffff:ffff:ffff:ffff|测试归属地3
|
|
||||||
2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
#### 测试九: 测试将多个IP数据并成一个IP数据
|
|
||||||
```
|
|
||||||
$ cat 1.txt
|
|
||||||
2001:200:123::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
$ ./bin/edit_v6
|
|
||||||
took: 1.99s
|
|
||||||
git diff ../../data
|
|
||||||
diff --git a/data/ipv6_source.txt b/data/ipv6_source.txt
|
|
||||||
index 4dee31b..ecd29c3 100644
|
|
||||||
--- a/data/ipv6_source.txt
|
|
||||||
+++ b/data/ipv6_source.txt
|
|
||||||
@@ -3,9 +3,7 @@
|
|
||||||
2001:200:102::|2001:200:104:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:105::|2001:200:11f:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:120::|2001:200:122:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:123::|2001:200:123:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
-2001:200:124::|2001:200:129:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
-2001:200:12a::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
+2001:200:123::|2001:200:12a:ffff:ffff:ffff:ffff:ffff|测试归属地
|
|
||||||
2001:200:12b::|2001:200:130:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
2001:200:131::|2001:200:132:ffff:ffff:ffff:ffff:ffff|日本|神奈川县|藤泽市|专线用户
|
|
||||||
2001:200:133::|2001:200:135:ffff:ffff:ffff:ffff:ffff|日本|东京都|千代田区|专线用户
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
@ -12,9 +12,8 @@ void test_ipv4(int policy) {
|
||||||
std::cout << "测试 IPv4 " << prompt[policy];
|
std::cout << "测试 IPv4 " << prompt[policy];
|
||||||
|
|
||||||
xdb::search_t s("../../data/ip2region_v4.xdb", xdb::ipv4, policy);
|
xdb::search_t s("../../data/ip2region_v4.xdb", xdb::ipv4, policy);
|
||||||
test(s, "0.0.0.0", "0|0|内网IP|内网IP");
|
test(s, "0.0.0.0", "Reserved|Reserved|Reserved|0|0");
|
||||||
test(s, "1.2.3.4", "美国|华盛顿|0|谷歌");
|
test(s, "1.2.3.4", "Australia|Queensland|Brisbane|0|AU");
|
||||||
test(s, "255.255.255.255", "0|0|内网IP|内网IP");
|
|
||||||
|
|
||||||
std::cout << " 成功" << std::endl;
|
std::cout << " 成功" << std::endl;
|
||||||
}
|
}
|
||||||
|
|
@ -24,7 +23,9 @@ void test_ipv6(int policy) {
|
||||||
|
|
||||||
xdb::search_t s("../../data/ip2region_v6.xdb", xdb::ipv6, policy);
|
xdb::search_t s("../../data/ip2region_v6.xdb", xdb::ipv6, policy);
|
||||||
test(s, "::1", "");
|
test(s, "::1", "");
|
||||||
test(s, "2001:200:124::", "日本|东京都|千代田区|专线用户");
|
test(s, "2001:200:124::", "Japan|Tokyo|Asagaya-minami|WIDE Project|JP");
|
||||||
|
test(s, "2001:200:124::", "Japan|Tokyo|Asagaya-minami|WIDE Project|JP");
|
||||||
|
test(s, "240e:3b7:3273:51d0:cd38:8ae1:e3c0:b708", "中国|广东省|深圳市|电信|CN");
|
||||||
|
|
||||||
std::cout << " 成功" << std::endl;
|
std::cout << " 成功" << std::endl;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,13 +19,13 @@ public class SearcherTest
|
||||||
private readonly string _xdbPathV6 = Path.Combine(AppContext.BaseDirectory, "TestData", "ip2region_v6.xdb");
|
private readonly string _xdbPathV6 = Path.Combine(AppContext.BaseDirectory, "TestData", "ip2region_v6.xdb");
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
[InlineData("58.251.27.201", "中国|广东省|深圳市|联通", "v4")]
|
[InlineData("58.251.27.201", "中国|广东省|深圳市|联通|CN", "v4")]
|
||||||
[InlineData("114.114.114.114", "中国|江苏省|南京市|0", "v4")]
|
[InlineData("114.114.114.114", "中国|江苏省|南京市|0|CN", "v4")]
|
||||||
[InlineData("119.29.29.29", "中国|北京|北京市|腾讯", "v4")]
|
[InlineData("119.29.29.29", "中国|北京|北京市|腾讯|CN", "v4")]
|
||||||
[InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里云", "v4")]
|
[InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里|CN", "v4")]
|
||||||
[InlineData("180.76.76.76", "中国|北京|北京市|百度", "v4")]
|
[InlineData("180.76.76.76", "中国|北京|北京市|百度|CN", "v4")]
|
||||||
[InlineData("8.8.8.8", "美国|0|0|Level3", "v4")]
|
[InlineData("8.8.8.8", "United States|California|0|Google LLC|US", "v4")]
|
||||||
[InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|家庭宽带", "v6")]
|
[InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|电信|CN", "v6")]
|
||||||
public void TestSearchCacheContent(string ip, string expected, string version)
|
public void TestSearchCacheContent(string ip, string expected, string version)
|
||||||
{
|
{
|
||||||
var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6;
|
var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6;
|
||||||
|
|
@ -35,13 +35,13 @@ public class SearcherTest
|
||||||
}
|
}
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
[InlineData("58.251.27.201", "中国|广东省|深圳市|联通", "v4")]
|
[InlineData("58.251.27.201", "中国|广东省|深圳市|联通|CN", "v4")]
|
||||||
[InlineData("114.114.114.114", "中国|江苏省|南京市|0", "v4")]
|
[InlineData("114.114.114.114", "中国|江苏省|南京市|0|CN", "v4")]
|
||||||
[InlineData("119.29.29.29", "中国|北京|北京市|腾讯", "v4")]
|
[InlineData("119.29.29.29", "中国|北京|北京市|腾讯|CN", "v4")]
|
||||||
[InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里云", "v4")]
|
[InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里|CN", "v4")]
|
||||||
[InlineData("180.76.76.76", "中国|北京|北京市|百度", "v4")]
|
[InlineData("180.76.76.76", "中国|北京|北京市|百度|CN", "v4")]
|
||||||
[InlineData("8.8.8.8", "美国|0|0|Level3", "v4")]
|
[InlineData("8.8.8.8", "United States|California|0|Google LLC|US", "v4")]
|
||||||
[InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|家庭宽带", "v6")]
|
[InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|电信|CN", "v6")]
|
||||||
public void TestSearchCacheVector(string ip, string expected, string version)
|
public void TestSearchCacheVector(string ip, string expected, string version)
|
||||||
{
|
{
|
||||||
var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6;
|
var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6;
|
||||||
|
|
@ -51,16 +51,16 @@ public class SearcherTest
|
||||||
}
|
}
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
[InlineData("58.251.0.0", "中国|广东省|深圳市|联通", "v4")]
|
[InlineData("58.251.0.0", "中国|广东省|深圳市|联通|CN", "v4")]
|
||||||
[InlineData("58.251.255.255", "中国|广东省|深圳市|联通", "v4")]
|
[InlineData("58.251.255.255", "中国|广东省|深圳市|联通|CN", "v4")]
|
||||||
[InlineData("58.251.27.201", "中国|广东省|深圳市|联通", "v4")]
|
[InlineData("58.251.27.201", "中国|广东省|深圳市|联通|CN", "v4")]
|
||||||
[InlineData("114.114.114.114", "中国|江苏省|南京市|0", "v4")]
|
[InlineData("114.114.114.114", "中国|江苏省|南京市|0|CN", "v4")]
|
||||||
[InlineData("119.29.29.29", "中国|北京|北京市|腾讯", "v4")]
|
[InlineData("119.29.29.29", "中国|北京|北京市|腾讯|CN", "v4")]
|
||||||
[InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里云", "v4")]
|
[InlineData("223.5.5.5", "中国|浙江省|杭州市|阿里|CN", "v4")]
|
||||||
[InlineData("180.76.76.76", "中国|北京|北京市|百度", "v4")]
|
[InlineData("180.76.76.76", "中国|北京|北京市|百度|CN", "v4")]
|
||||||
[InlineData("8.8.8.8", "美国|0|0|Level3", "v4")]
|
[InlineData("8.8.8.8", "United States|California|0|Google LLC|US", "v4")]
|
||||||
[InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|家庭宽带", "v6")]
|
[InlineData("240e:3b7:3272:d8d0:db09:c067:8d59:539e", "中国|广东省|深圳市|电信|CN", "v6")]
|
||||||
[InlineData("240e:044d:2d00:0000:0000:0000:0000:0000", "美国|加利福尼亚州|洛杉矶|移动网络", "v6")]
|
[InlineData("240e:044d:2d00:0000:0000:0000:0000:0000", "中国|云南|楚雄|电信|CN", "v6")]
|
||||||
public void TestSearchCacheFile(string ip, string expected, string version)
|
public void TestSearchCacheFile(string ip, string expected, string version)
|
||||||
{
|
{
|
||||||
var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6;
|
var _xdbPath = version == "v4" ? _xdbPathV4 : _xdbPathV6;
|
||||||
|
|
@ -109,7 +109,7 @@ public class SearcherTest
|
||||||
}
|
}
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
[InlineData("58.251.255.255", "中国|广东省|深圳市|联通")]
|
[InlineData("58.251.255.255", "中国|广东省|深圳市|联通|CN")]
|
||||||
public void Search_Ip_Ok(string ipStr, string expected)
|
public void Search_Ip_Ok(string ipStr, string expected)
|
||||||
{
|
{
|
||||||
var fileSearcher = new Searcher(CachePolicy.File, _xdbPathV4);
|
var fileSearcher = new Searcher(CachePolicy.File, _xdbPathV4);
|
||||||
|
|
@ -119,7 +119,7 @@ public class SearcherTest
|
||||||
}
|
}
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
[InlineData("58.251.255.255", "中国|广东省|深圳市|联通")]
|
[InlineData("58.251.255.255", "中国|广东省|深圳市|联通|CN")]
|
||||||
public void AddIP2RegionService_Ok(string ipStr, string expected)
|
public void AddIP2RegionService_Ok(string ipStr, string expected)
|
||||||
{
|
{
|
||||||
var services = new ServiceCollection();
|
var services = new ServiceCollection();
|
||||||
|
|
|
||||||
|
|
@ -19,9 +19,9 @@ public class XdbTest
|
||||||
var version = await XDB.Util.GetVersionAsync(db);
|
var version = await XDB.Util.GetVersionAsync(db);
|
||||||
Assert.Equal(3, version.Ver);
|
Assert.Equal(3, version.Ver);
|
||||||
Assert.Equal(1, version.CachePolice);
|
Assert.Equal(1, version.CachePolice);
|
||||||
Assert.Equal("2025-09-06 02:24:16", version.CreatedTime.ToString("yyyy-MM-dd HH:mm:ss"));
|
//Assert.Equal("2025-09-06 02:24:16", version.CreatedTime.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
Assert.Equal((uint)955933, version.StartIndex);
|
//Assert.Equal((uint)955933, version.StartIndex);
|
||||||
Assert.Equal((uint)11042415, version.EndIndex);
|
//Assert.Equal((uint)11042415, version.EndIndex);
|
||||||
Assert.Equal(4, version.IPVer);
|
Assert.Equal(4, version.IPVer);
|
||||||
Assert.Equal(4, version.BytesCount);
|
Assert.Equal(4, version.BytesCount);
|
||||||
}
|
}
|
||||||
|
|
@ -33,9 +33,9 @@ public class XdbTest
|
||||||
var version = await XDB.Util.GetVersionAsync(db);
|
var version = await XDB.Util.GetVersionAsync(db);
|
||||||
Assert.Equal(3, version.Ver);
|
Assert.Equal(3, version.Ver);
|
||||||
Assert.Equal(1, version.CachePolice);
|
Assert.Equal(1, version.CachePolice);
|
||||||
Assert.Equal("2025-10-17 04:41:04", version.CreatedTime.ToString("yyyy-MM-dd HH:mm:ss"));
|
//Assert.Equal("2025-10-17 04:41:04", version.CreatedTime.ToString("yyyy-MM-dd HH:mm:ss"));
|
||||||
Assert.Equal((uint)3094259, version.StartIndex);
|
//Assert.Equal((uint)3094259, version.StartIndex);
|
||||||
Assert.Equal((uint)36258303, version.EndIndex);
|
//Assert.Equal((uint)36258303, version.EndIndex);
|
||||||
Assert.Equal(6, version.IPVer);
|
Assert.Equal(6, version.IPVer);
|
||||||
Assert.Equal(4, version.BytesCount);
|
Assert.Equal(4, version.BytesCount);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<id>IP2Region.Net</id>
|
<id>IP2Region.Net</id>
|
||||||
<version>3.0.0</version>
|
<version>3.0.2</version>
|
||||||
<title>IP2Region.Net</title>
|
<title>IP2Region.Net</title>
|
||||||
<authors>Alan Lee;Argo Zhang(argo@live.ca)</authors>
|
<authors>Alan Lee;Argo Zhang(argo@live.ca)</authors>
|
||||||
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
|
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
|
||||||
|
|
|
||||||
|
|
@ -75,6 +75,14 @@ public class Searcher(CachePolicy cachePolicy, string xdbPath) : ISearcher
|
||||||
var sPtr = BinaryPrimitives.ReadUInt32LittleEndian(vector.Span);
|
var sPtr = BinaryPrimitives.ReadUInt32LittleEndian(vector.Span);
|
||||||
var ePtr = BinaryPrimitives.ReadUInt32LittleEndian(vector.Span.Slice(4));
|
var ePtr = BinaryPrimitives.ReadUInt32LittleEndian(vector.Span.Slice(4));
|
||||||
|
|
||||||
|
// @Note: ptr validate, zero ptr means source data missing
|
||||||
|
// so we could just stop here and return an empty string.
|
||||||
|
if (sPtr == 0 || ePtr == 0)
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
var length = ipBytes.Length;
|
var length = ipBytes.Length;
|
||||||
var indexSize = length * 2 + 6;
|
var indexSize = length * 2 + 6;
|
||||||
var l = 0;
|
var l = 0;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,18 @@
|
||||||
|
.PHONY: all compile test clean bench-v4 bench-v6
|
||||||
|
|
||||||
|
all: compile
|
||||||
|
|
||||||
|
compile:
|
||||||
|
rebar3 compile
|
||||||
|
|
||||||
|
test:
|
||||||
|
rebar3 eunit
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rebar3 clean
|
||||||
|
|
||||||
|
bench-v4:
|
||||||
|
sh benchmarks/xdb-benchmark.sh ipv4
|
||||||
|
|
||||||
|
bench-v6:
|
||||||
|
sh benchmarks/xdb-benchmark.sh ipv6
|
||||||
|
|
@ -1,129 +1,180 @@
|
||||||
# ip2region xdb erlang 查询客户端
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
### 简介
|
# ip2region Erlang query client
|
||||||
该bingding以erlang语言实现xdb查询客户端,基于Erlang OTP Application,查询逻辑由ip2region_worker工作进程实现,支持配多个工作进程来进行负载均衡。
|
|
||||||
|
|
||||||
### 应用配置
|
### Introduction
|
||||||
该应用可配置的参数在ip2region.app.src中,如下:
|
|
||||||
``` erlang
|
This binding implements the xdb query client in `Erlang`, based on the `Erlang OTP Application`. The query logic is implemented by the `ip2region_worker` worker process, supporting multiple worker processes for load balancing.
|
||||||
|
|
||||||
|
### Application Configuration
|
||||||
|
|
||||||
|
The configurable parameters for this application are in `ip2region.app.src`, as follows:
|
||||||
|
|
||||||
|
```erlang
|
||||||
{env,[
|
{env,[
|
||||||
{poolargs, [
|
{poolargs, [
|
||||||
{size, 1}, %% 工作进程默认数量
|
{size, 1}, %% Default number of worker processes
|
||||||
{max_overflow, 5} %% 工作进程最大数量
|
{max_overflow, 5} %% Maximum number of worker processes
|
||||||
|
]},
|
||||||
|
{db, [
|
||||||
|
{ipv4, "ip2region.xdb"} %% Default IPv4 xdb file
|
||||||
]}
|
]}
|
||||||
]}
|
]}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 编译
|
### Dual-stack configuration (IPv4 + IPv6)
|
||||||
|
|
||||||
|
To enable IPv6 queries, add the `ipv6` entry to the `db` list and place both xdb files under `priv/`:
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
{env,[
|
||||||
|
{poolargs, [
|
||||||
|
{size, 1},
|
||||||
|
{max_overflow, 5}
|
||||||
|
]},
|
||||||
|
{db, [
|
||||||
|
{ipv4, "ip2region.xdb"},
|
||||||
|
{ipv6, "ip2region_v6.xdb"}
|
||||||
|
]}
|
||||||
|
]}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The `xdb:search/1` interface automatically detects IPv4 and IPv6 inputs and routes them to the correct worker pool.
|
||||||
|
|
||||||
|
### Compile
|
||||||
|
|
||||||
|
```bash
|
||||||
$ rebar3 compile
|
$ rebar3 compile
|
||||||
```
|
```
|
||||||
### 运行
|
|
||||||
将xdb文件放到priv目录下,然后启动erlang节点:
|
|
||||||
```
|
|
||||||
$ rebar3 shell
|
|
||||||
|
|
||||||
|
### Run
|
||||||
|
|
||||||
|
Place the xdb file in the `priv` directory, then start the Erlang node:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ rebar3 shell
|
||||||
```
|
```
|
||||||
在erlang shell中调用xdb:search/1接口查询Ip地址信息, 该接口支持以list格式字符串、binary格式字符串、tuple和整数表示的IP地址,如下:
|
|
||||||
```
|
Call the `xdb:search/1` interface in the Erlang shell to query IP address information. This interface supports IP addresses represented as list strings, binary strings, tuples, and integers:
|
||||||
|
|
||||||
|
```erlang
|
||||||
1> xdb:search("1.0.8.0").
|
1> xdb:search("1.0.8.0").
|
||||||
[20013,22269,124,48,124,24191,19996,30465,124,24191,24030,
|
[20013,22269,124,24191,19996,30465,124,24191,24030,24066,
|
||||||
24066,124,30005,20449]
|
124,20013,22269,30005,20449,124,67,78]
|
||||||
2>
|
2>
|
||||||
3> io:format("~ts~n", [xdb:search("1.0.8.0")]).
|
3> io:format("~ts~n", [xdb:search("1.0.8.0")]).
|
||||||
中国|0|广东省|广州市|电信
|
中国|广东省|广州市|中国电信|CN
|
||||||
io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]).
|
4> io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]).
|
||||||
中国|0|广东省|广州市|电信
|
中国|广东省|广州市|中国电信|CN
|
||||||
4> io:format("~ts~n", [xdb:search({1,0,8,0})]).
|
5> io:format("~ts~n", [xdb:search({1,0,8,0})]).
|
||||||
中国|0|广东省|广州市|电信
|
中国|广东省|广州市|中国电信|CN
|
||||||
6> io:format("~ts~n", [xdb:search(16779264)]).
|
6> io:format("~ts~n", [xdb:search(16779264)]).
|
||||||
中国|0|广东省|广州市|电信
|
中国|广东省|广州市|中国电信|CN
|
||||||
```
|
```
|
||||||
|
|
||||||
### 使用方法
|
With dual-stack enabled, IPv6 addresses are supported in the same way:
|
||||||
* 在rebar.config中引入依赖
|
|
||||||
|
```erlang
|
||||||
|
1> io:format("~ts~n", [xdb:search("2001:4860:4860::8888")]).
|
||||||
|
United States|Florida|Miami|Google LLC|US
|
||||||
|
2> io:format("~ts~n", [xdb:search(<<"2001:4860:4860::8888">>)]).
|
||||||
|
United States|Florida|Miami|Google LLC|US
|
||||||
|
3> io:format("~ts~n", [xdb:search({8193,18528,18528,0,0,0,0,34952})]).
|
||||||
|
United States|Florida|Miami|Google LLC|US
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
|
||||||
|
- Add the dependency in `rebar.config`
|
||||||
|
|
||||||
|
```erlang
|
||||||
{deps, [
|
{deps, [
|
||||||
ip2region
|
ip2region
|
||||||
]}.
|
]}.
|
||||||
|
|
||||||
```
|
|
||||||
* 启动ip2region Application
|
|
||||||
```
|
|
||||||
......
|
|
||||||
|
|
||||||
application:ensure_started(ip2region),
|
|
||||||
|
|
||||||
......
|
|
||||||
```
|
```
|
||||||
|
|
||||||
* 调用xdb:search/1接口查询IP信息
|
- Start the ip2region Application
|
||||||
```
|
|
||||||
......
|
|
||||||
|
|
||||||
ip2region:search("1.0.8.0"),
|
```erlang
|
||||||
|
{ok, _} = application:ensure_all_started(ip2region).
|
||||||
......
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 单元测试
|
- Call the `xdb:search/1` interface to query IP information
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
xdb:search("1.0.8.0").
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Unit Test
|
||||||
|
|
||||||
|
```bash
|
||||||
$ rebar3 eunit
|
$ rebar3 eunit
|
||||||
===> Verifying dependencies...
|
===> Verifying dependencies...
|
||||||
===> Analyzing applications...
|
===> Analyzing applications...
|
||||||
===> Compiling ip2region
|
===> Compiling ip2region
|
||||||
===> Performing EUnit tests...
|
===> Performing EUnit tests...
|
||||||
=INFO REPORT==== 17-Jan-2023::11:52:59.920155 ===
|
=INFO REPORT==== 28-Jun-2026::04:53:28 ===
|
||||||
XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb
|
XdbFile:/Users/nana/Documents/code/ip2region/.worktrees/erlang-ipv6/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb
|
||||||
|
|
||||||
....
|
....
|
||||||
Finished in 0.074 seconds
|
Finished in 0.192 seconds
|
||||||
4 tests, 0 failures
|
63 tests, 0 failures
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 基准测试
|
### Benchmark
|
||||||
```
|
|
||||||
|
Both IPv4 and IPv6 benchmarks share the same script. Run it with the desired IP version:
|
||||||
|
|
||||||
|
> `cold` = first pass over the source file: each IP triggers a real search and the
|
||||||
|
> result is written into the ETS cache. `warm` = second pass over the same list,
|
||||||
|
> where every lookup is served directly from the ETS cache.
|
||||||
|
|
||||||
|
```bash
|
||||||
$ cd benchmarks/
|
$ cd benchmarks/
|
||||||
$ sh xdb-benchmark.sh
|
$ sh xdb-benchmark.sh ipv4
|
||||||
===> Verifying dependencies...
|
```
|
||||||
===> Analyzing applications...
|
|
||||||
===> Compiling ip2region
|
For IPv6:
|
||||||
Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit]
|
|
||||||
|
```bash
|
||||||
Eshell V12.3.2.2 (abort with ^G)
|
$ sh xdb-benchmark.sh ipv6
|
||||||
1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 ===
|
```
|
||||||
XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb
|
|
||||||
|
Or use the Makefile targets from the `binding/erlang` directory:
|
||||||
===> Booted ip2region
|
|
||||||
===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()."
|
```bash
|
||||||
CPU info:
|
$ make bench-v4
|
||||||
model name : AMD EPYC 7K62 48-Core Processor
|
$ make bench-v6
|
||||||
cache size : 512 KB
|
```
|
||||||
cpu MHz : 2595.124
|
|
||||||
bogomips : 5190.24
|
#### IPv4 benchmark example
|
||||||
cores/threads : 2
|
|
||||||
|
```bash
|
||||||
Erlang info:
|
System:
|
||||||
system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit]
|
CPU : Apple M4
|
||||||
load test data use 4.835593s
|
Cores : 10 cores / 10 threads
|
||||||
|
Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace]
|
||||||
start run benchmark tests
|
Loaded : 487169 IPs in 1.335 s
|
||||||
|
|
||||||
search from file:
|
Benchmarks:
|
||||||
ip count:683844,
|
cold total= 9.601s count= 487169 qps= 50740.66 avg= 0.019708 ms/op (19.708 us/op)
|
||||||
total time: 28.201699s,
|
warm total= 0.160s count= 487169 qps= 3053164.29 avg= 0.000328 ms/op ( 0.328 us/op)
|
||||||
search 24248.326315375536 times per second,
|
|
||||||
use 41.23995969841075 micro second per search
|
Done.
|
||||||
|
```
|
||||||
search from cache:
|
|
||||||
ip count:683844,
|
#### IPv6 benchmark example
|
||||||
total time: 0.671801s,
|
|
||||||
search 1017926.4395259906 times per second,
|
```bash
|
||||||
use 0.9823892583688677 micro second per search
|
System:
|
||||||
|
CPU : Apple M4
|
||||||
benchmark test finish
|
Cores : 10 cores / 10 threads
|
||||||
|
Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace]
|
||||||
|
Loaded : 638953 IPs in 2.949 s
|
||||||
|
|
||||||
|
Benchmarks:
|
||||||
|
cold total= 20.504s count= 638953 qps= 31162.52 avg= 0.032090 ms/op (32.090 us/op)
|
||||||
|
warm total= 0.444s count= 638953 qps= 1437781.56 avg= 0.000696 ms/op ( 0.696 us/op)
|
||||||
|
|
||||||
|
Done.
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,179 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region Erlang 查询客户端
|
||||||
|
|
||||||
|
### 简介
|
||||||
|
|
||||||
|
该 bingding 以 `Erlang` 语言实现 xdb 查询客户端,基于 `Erlang OTP Application`,查询逻辑由 `ip2region_worker` 工作进程实现,支持配多个工作进程来进行负载均衡。
|
||||||
|
|
||||||
|
### 应用配置
|
||||||
|
|
||||||
|
该应用可配置的参数在 `ip2region.app.src` 中, 如下:
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
{env,[
|
||||||
|
{poolargs, [
|
||||||
|
{size, 1}, %% 工作进程默认数量
|
||||||
|
{max_overflow, 5} %% 工作进程最大数量
|
||||||
|
]},
|
||||||
|
{db, [
|
||||||
|
{ipv4, "ip2region.xdb"} %% 默认 IPv4 xdb 文件
|
||||||
|
]}
|
||||||
|
]}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 双栈配置(IPv4 + IPv6)
|
||||||
|
|
||||||
|
如需启用 IPv6 查询,在 `db` 列表中加入 `ipv6` 项,并将两个 xdb 文件放到 `priv/` 目录下:
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
{env,[
|
||||||
|
{poolargs, [
|
||||||
|
{size, 1},
|
||||||
|
{max_overflow, 5}
|
||||||
|
]},
|
||||||
|
{db, [
|
||||||
|
{ipv4, "ip2region.xdb"},
|
||||||
|
{ipv6, "ip2region_v6.xdb"}
|
||||||
|
]}
|
||||||
|
]}
|
||||||
|
```
|
||||||
|
|
||||||
|
`xdb:search/1` 接口会自动识别 IPv4 与 IPv6 输入,并将其路由到对应的工作进程池。
|
||||||
|
|
||||||
|
### 编译
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ rebar3 compile
|
||||||
|
```
|
||||||
|
|
||||||
|
### 运行
|
||||||
|
|
||||||
|
将 xdb 文件放到 `priv` 目录下,然后启动 Erlang 节点:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ rebar3 shell
|
||||||
|
```
|
||||||
|
|
||||||
|
在 Erlang shell 中调用 `xdb:search/1` 接口查询 IP 地址信息。该接口支持 list 字符串、binary 字符串、tuple 和整数表示的 IP 地址:
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
1> xdb:search("1.0.8.0").
|
||||||
|
[20013,22269,124,24191,19996,30465,124,24191,24030,24066,
|
||||||
|
124,20013,22269,30005,20449,124,67,78]
|
||||||
|
2>
|
||||||
|
3> io:format("~ts~n", [xdb:search("1.0.8.0")]).
|
||||||
|
中国|广东省|广州市|中国电信|CN
|
||||||
|
4> io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]).
|
||||||
|
中国|广东省|广州市|中国电信|CN
|
||||||
|
5> io:format("~ts~n", [xdb:search({1,0,8,0})]).
|
||||||
|
中国|广东省|广州市|中国电信|CN
|
||||||
|
6> io:format("~ts~n", [xdb:search(16779264)]).
|
||||||
|
中国|广东省|广州市|中国电信|CN
|
||||||
|
```
|
||||||
|
|
||||||
|
启用双栈后,IPv6 地址也按同样方式支持:
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
1> io:format("~ts~n", [xdb:search("2001:4860:4860::8888")]).
|
||||||
|
United States|Florida|Miami|Google LLC|US
|
||||||
|
2> io:format("~ts~n", [xdb:search(<<"2001:4860:4860::8888">>)]).
|
||||||
|
United States|Florida|Miami|Google LLC|US
|
||||||
|
3> io:format("~ts~n", [xdb:search({8193,18528,18528,0,0,0,0,34952})]).
|
||||||
|
United States|Florida|Miami|Google LLC|US
|
||||||
|
```
|
||||||
|
|
||||||
|
### 使用方法
|
||||||
|
|
||||||
|
- 在 `rebar.config` 中引入依赖
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
{deps, [
|
||||||
|
ip2region
|
||||||
|
]}.
|
||||||
|
```
|
||||||
|
|
||||||
|
- 启动 ip2region Application
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
{ok, _} = application:ensure_all_started(ip2region).
|
||||||
|
```
|
||||||
|
|
||||||
|
- 调用 `xdb:search/1` 接口查询 IP 信息
|
||||||
|
|
||||||
|
```erlang
|
||||||
|
xdb:search("1.0.8.0").
|
||||||
|
```
|
||||||
|
|
||||||
|
### 单元测试
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ rebar3 eunit
|
||||||
|
===> Verifying dependencies...
|
||||||
|
===> Analyzing applications...
|
||||||
|
===> Compiling ip2region
|
||||||
|
===> Performing EUnit tests...
|
||||||
|
=INFO REPORT==== 28-Jun-2026::04:53:28 ===
|
||||||
|
XdbFile:/Users/nana/Documents/code/ip2region/.worktrees/erlang-ipv6/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb
|
||||||
|
|
||||||
|
....
|
||||||
|
Finished in 0.192 seconds
|
||||||
|
63 tests, 0 failures
|
||||||
|
```
|
||||||
|
|
||||||
|
### 基准测试
|
||||||
|
|
||||||
|
IPv4 与 IPv6 共用一个脚本,通过参数指定版本:
|
||||||
|
|
||||||
|
> `cold` = 第一次遍历源文件:每个 IP 都会触发真实查询,并把结果写入 ETS 缓存。
|
||||||
|
> `warm` = 第二次遍历同一列表,所有查询都直接命中 ETS 缓存。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ cd benchmarks/
|
||||||
|
$ sh xdb-benchmark.sh ipv4
|
||||||
|
```
|
||||||
|
|
||||||
|
IPv6:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ sh xdb-benchmark.sh ipv6
|
||||||
|
```
|
||||||
|
|
||||||
|
也可以在 `binding/erlang` 目录直接用 Makefile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ make bench-v4
|
||||||
|
$ make bench-v6
|
||||||
|
```
|
||||||
|
|
||||||
|
#### IPv4 基准测试示例
|
||||||
|
|
||||||
|
```bash
|
||||||
|
System:
|
||||||
|
CPU : Apple M4
|
||||||
|
Cores : 10 cores / 10 threads
|
||||||
|
Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace]
|
||||||
|
Loaded : 487169 IPs in 1.335 s
|
||||||
|
|
||||||
|
Benchmarks:
|
||||||
|
cold total= 9.601s count= 487169 qps= 50740.66 avg= 0.019708 ms/op (19.708 us/op)
|
||||||
|
warm total= 0.160s count= 487169 qps= 3053164.29 avg= 0.000328 ms/op ( 0.328 us/op)
|
||||||
|
|
||||||
|
Done.
|
||||||
|
```
|
||||||
|
|
||||||
|
#### IPv6 基准测试示例
|
||||||
|
|
||||||
|
```bash
|
||||||
|
System:
|
||||||
|
CPU : Apple M4
|
||||||
|
Cores : 10 cores / 10 threads
|
||||||
|
Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace]
|
||||||
|
Loaded : 638953 IPs in 2.949 s
|
||||||
|
|
||||||
|
Benchmarks:
|
||||||
|
cold total= 20.504s count= 638953 qps= 31162.52 avg= 0.032090 ms/op (32.090 us/op)
|
||||||
|
warm total= 0.444s count= 638953 qps= 1437781.56 avg= 0.000696 ms/op ( 0.696 us/op)
|
||||||
|
|
||||||
|
Done.
|
||||||
|
```
|
||||||
|
|
@ -1,5 +1,37 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
cd ..
|
# Run from binding/erlang so paths and rebar3 artifacts are correct.
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
rebar3 shell --eval="xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()."
|
VERSION="${1:-ipv4}"
|
||||||
|
PROJECT_ROOT="$(cd ../.. && pwd)"
|
||||||
|
|
||||||
|
if [ "$VERSION" = "ipv6" ]; then
|
||||||
|
DB_FILE="${PROJECT_ROOT}/data/ip2region_v6.xdb"
|
||||||
|
DATA_FILE="${PROJECT_ROOT}/data/ipv6_source.txt"
|
||||||
|
DB_CONFIG="[{ipv6, \"${DB_FILE}\"}]"
|
||||||
|
elif [ "$VERSION" = "ipv4" ]; then
|
||||||
|
DATA_FILE="${PROJECT_ROOT}/data/ipv4_source.txt"
|
||||||
|
DB_CONFIG="[{ipv4, \"ip2region.xdb\"}]"
|
||||||
|
else
|
||||||
|
echo "Usage: $0 [ipv4|ipv6]"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
rebar3 compile
|
||||||
|
|
||||||
|
# shellcheck disable=SC2089
|
||||||
|
EVAL="application:load(ip2region), \
|
||||||
|
application:set_env(ip2region, db, ${DB_CONFIG}), \
|
||||||
|
application:set_env(ip2region, poolargs, [{size,4},{max_overflow,8}]), \
|
||||||
|
{ok,_}=application:ensure_all_started(ip2region), \
|
||||||
|
xdb_benchmark:main(\"${DATA_FILE}\"), \
|
||||||
|
init:stop()."
|
||||||
|
|
||||||
|
# shellcheck disable=SC2090
|
||||||
|
erl -pa _build/default/lib/poolboy/ebin \
|
||||||
|
-pa _build/default/lib/ip2region/ebin \
|
||||||
|
-kernel logger_level error \
|
||||||
|
-noshell \
|
||||||
|
-eval "${EVAL}"
|
||||||
|
|
|
||||||
|
|
@ -4,23 +4,39 @@
|
||||||
-define(NONE, none).
|
-define(NONE, none).
|
||||||
-define(APP_NAME, ip2region).
|
-define(APP_NAME, ip2region).
|
||||||
|
|
||||||
-define(XDB_VECTOR_INDEX, ets_xdb_vector_index).
|
|
||||||
-define(XDB_SEGMENT_INDEX, ets_xdb_segement_index).
|
|
||||||
-define(IP2REGION_CACHE, ets_ip2region_cache).
|
|
||||||
|
|
||||||
|
|
||||||
-define(XDB_HEADER_SIZE, 256).
|
-define(XDB_HEADER_SIZE, 256).
|
||||||
-define(XDB_VECTOR_COLS, 256).
|
-define(XDB_VECTOR_COLS, 256).
|
||||||
-define(XDB_VECTOR_INDEX_SIZE, 8).
|
-define(XDB_VECTOR_INDEX_SIZE, 8).
|
||||||
-define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256
|
-define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256
|
||||||
|
|
||||||
-define(XDB_SEGMENT_INDEX_SIZE, 14).
|
%% IP versions as stored in xdb header
|
||||||
|
-define(IP_VERSION_4, 4).
|
||||||
|
-define(IP_VERSION_6, 6).
|
||||||
|
|
||||||
-define(IP2REGION_POOL, ip2region_pool).
|
%% Segment index sizes (bytes): start_ip + end_ip + data_len + data_ptr
|
||||||
|
-define(XDB_SEGMENT_INDEX_SIZE_V4, 14). %% 4 + 4 + 2 + 4
|
||||||
|
-define(XDB_SEGMENT_INDEX_SIZE_V6, 38). %% 16 + 16 + 2 + 4
|
||||||
|
|
||||||
-ifndef(IF).
|
%% Per-version ETS tables
|
||||||
-define(IF(C, T, F), case (C) of true -> (T); false -> (F) end).
|
-define(XDB_VECTOR_INDEX_V4, ets_xdb_vector_index_v4).
|
||||||
-define(IF(C, T), ?IF(C, T, skip)).
|
-define(XDB_VECTOR_INDEX_V6, ets_xdb_vector_index_v6).
|
||||||
-endif.
|
-define(XDB_SEGMENT_INDEX_V4, ets_xdb_segment_index_v4).
|
||||||
|
-define(XDB_SEGMENT_INDEX_V6, ets_xdb_segment_index_v6).
|
||||||
|
-define(IP2REGION_CACHE_V4, ets_ip2region_cache_v4).
|
||||||
|
-define(IP2REGION_CACHE_V6, ets_ip2region_cache_v6).
|
||||||
|
|
||||||
|
%% Per-version pool names.
|
||||||
|
-define(IP2REGION_POOL_V4, ip2region_pool_v4).
|
||||||
|
-define(IP2REGION_POOL_V6, ip2region_pool_v6).
|
||||||
|
|
||||||
|
-record(xdb_header, {
|
||||||
|
version :: non_neg_integer(),
|
||||||
|
index_policy :: non_neg_integer(),
|
||||||
|
created_at :: non_neg_integer(),
|
||||||
|
start_index_ptr :: non_neg_integer(),
|
||||||
|
end_index_ptr :: non_neg_integer(),
|
||||||
|
ip_version :: non_neg_integer(),
|
||||||
|
runtime_ptr_bytes :: non_neg_integer()
|
||||||
|
}).
|
||||||
|
|
||||||
-endif.
|
-endif.
|
||||||
|
|
@ -18,7 +18,7 @@
|
||||||
{ex_doc, [
|
{ex_doc, [
|
||||||
{extras, ["README.md"]},
|
{extras, ["README.md"]},
|
||||||
{main, "README.md"},
|
{main, "README.md"},
|
||||||
{source_url, "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}
|
{source_url, "https://github.com/lionsoul2014/ip2region/tree/master/binding/erlang"}
|
||||||
]}.
|
]}.
|
||||||
|
|
||||||
{hex, [{doc, ex_doc}]}.
|
{hex, [{doc, ex_doc}]}.
|
||||||
|
|
|
||||||
|
|
@ -5,16 +5,20 @@
|
||||||
{mod, {ip2region_app, []}},
|
{mod, {ip2region_app, []}},
|
||||||
{applications,
|
{applications,
|
||||||
[kernel,
|
[kernel,
|
||||||
stdlib
|
stdlib,
|
||||||
|
poolboy
|
||||||
]},
|
]},
|
||||||
{env,[
|
{env,[
|
||||||
{poolargs, [
|
{poolargs, [
|
||||||
{size, 1},
|
{size, 1},
|
||||||
{max_overflow, 5}
|
{max_overflow, 5}
|
||||||
|
]},
|
||||||
|
{db, [
|
||||||
|
{ipv4, "ip2region.xdb"}
|
||||||
]}
|
]}
|
||||||
]},
|
]},
|
||||||
{modules, []},
|
{modules, []},
|
||||||
|
|
||||||
{licenses, ["Apache-2.0"]},
|
{licenses, ["Apache-2.0"]},
|
||||||
{links, [{"Github", "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}]}
|
{links, [{"Github", "https://github.com/lionsoul2014/ip2region/tree/master/binding/erlang"}]}
|
||||||
]}.
|
]}.
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,6 @@
|
||||||
|
|
||||||
start_link() ->
|
start_link() ->
|
||||||
{ok, SupPid} = supervisor:start_link({local, ?SERVER}, ?MODULE, []),
|
{ok, SupPid} = supervisor:start_link({local, ?SERVER}, ?MODULE, []),
|
||||||
{ok, _PoolPid} = start_ip2region_pool(SupPid),
|
|
||||||
{ok, SupPid}.
|
{ok, SupPid}.
|
||||||
|
|
||||||
%% sup_flags() = #{strategy => strategy(), % optional
|
%% sup_flags() = #{strategy => strategy(), % optional
|
||||||
|
|
@ -35,21 +34,45 @@ init([]) ->
|
||||||
SupFlags = #{strategy => one_for_one,
|
SupFlags = #{strategy => one_for_one,
|
||||||
intensity => 10,
|
intensity => 10,
|
||||||
period => 5},
|
period => 5},
|
||||||
ChildSpecs = [],
|
ChildSpecs = pool_child_specs(),
|
||||||
{ok, {SupFlags, ChildSpecs}}.
|
{ok, {SupFlags, ChildSpecs}}.
|
||||||
|
|
||||||
%% internal functions
|
%% internal functions
|
||||||
%%
|
%%
|
||||||
create_table() ->
|
create_table() ->
|
||||||
Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}],
|
Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}],
|
||||||
ets:new(?XDB_VECTOR_INDEX, Opts),
|
%% Version-specific tables for dual-stack support
|
||||||
ets:new(?XDB_SEGMENT_INDEX, Opts),
|
ensure_table(?XDB_VECTOR_INDEX_V4, Opts),
|
||||||
ets:new(?IP2REGION_CACHE, Opts).
|
ensure_table(?XDB_VECTOR_INDEX_V6, Opts),
|
||||||
|
ensure_table(?XDB_SEGMENT_INDEX_V4, Opts),
|
||||||
|
ensure_table(?XDB_SEGMENT_INDEX_V6, Opts),
|
||||||
|
ensure_table(?IP2REGION_CACHE_V4, Opts),
|
||||||
|
ensure_table(?IP2REGION_CACHE_V6, Opts).
|
||||||
|
|
||||||
start_ip2region_pool(Sup) ->
|
ensure_table(Name, Opts) ->
|
||||||
|
case ets:whereis(Name) of
|
||||||
|
undefined -> ets:new(Name, Opts);
|
||||||
|
_ -> ok
|
||||||
|
end.
|
||||||
|
|
||||||
|
pool_child_specs() ->
|
||||||
|
{ok, DbConfig} = application:get_env(db),
|
||||||
{ok, PoolArgsCfg} = application:get_env(poolargs),
|
{ok, PoolArgsCfg} = application:get_env(poolargs),
|
||||||
PoolName = ?IP2REGION_POOL,
|
lists:foldl(
|
||||||
PoolArgs = [{strategy, fifo}, {name, {local, PoolName}}, {worker_module, ip2region_worker} | PoolArgsCfg],
|
fun({ipv4, File}, Acc) ->
|
||||||
WorkerArgs = [],
|
[make_pool_spec(?IP2REGION_POOL_V4, ipv4, File, PoolArgsCfg) | Acc];
|
||||||
ChildSpecs = poolboy:child_spec(PoolName, PoolArgs, WorkerArgs),
|
({ipv6, File}, Acc) ->
|
||||||
supervisor:start_child(Sup, ChildSpecs).
|
[make_pool_spec(?IP2REGION_POOL_V6, ipv6, File, PoolArgsCfg) | Acc];
|
||||||
|
(_, Acc) ->
|
||||||
|
Acc
|
||||||
|
end, [], DbConfig).
|
||||||
|
|
||||||
|
make_pool_spec(PoolName, Version, File, PoolArgsCfg) ->
|
||||||
|
PoolArgs = [
|
||||||
|
{strategy, fifo},
|
||||||
|
{name, {local, PoolName}},
|
||||||
|
{worker_module, ip2region_worker}
|
||||||
|
| PoolArgsCfg
|
||||||
|
],
|
||||||
|
WorkerArgs = [{xdb_file, File}, {expected_version, Version}],
|
||||||
|
poolboy:child_spec(PoolName, PoolArgs, WorkerArgs).
|
||||||
|
|
|
||||||
|
|
@ -4,23 +4,68 @@
|
||||||
%% license that can be found in the LICENSE file.
|
%% license that can be found in the LICENSE file.
|
||||||
%%
|
%%
|
||||||
%% @doc
|
%% @doc
|
||||||
%% ip2region utils
|
%% ip2region utils: IPv4/IPv6 parsing and version detection.
|
||||||
%% @end
|
%% @end
|
||||||
%%%-------------------------------------------------------------------
|
%%%-------------------------------------------------------------------
|
||||||
-module(ip2region_util).
|
-module(ip2region_util).
|
||||||
-export([ipv4_to_n/1]).
|
|
||||||
|
|
||||||
|
-export([ipv4_to_n/1, ip_version/1, ip_to_bytes/1]).
|
||||||
|
|
||||||
ipv4_to_n(IntIp) when is_integer(IntIp) -> IntIp;
|
-define(IS_UINT8(V), is_integer(V), V >= 0, V =< 255).
|
||||||
ipv4_to_n({A, B, C, D}) ->
|
-define(IS_UINT16(V), is_integer(V), V >= 0, V =< 65535).
|
||||||
<<N:32>> = <<A, B, C, D>>,
|
|
||||||
N;
|
-spec ip_version(Ip :: tuple() | list() | binary() | integer()) ->
|
||||||
ipv4_to_n(Ip) when is_binary(Ip) ->
|
ipv4 | ipv6 | {error, atom()}.
|
||||||
ipv4_to_n(binary_to_list(Ip));
|
ip_version(Ip) when is_integer(Ip), Ip >= 0, Ip =< 16#FFFFFFFF -> ipv4;
|
||||||
ipv4_to_n(Ip) when is_list(Ip) ->
|
ip_version({A, B, C, D}) when ?IS_UINT8(A), ?IS_UINT8(B), ?IS_UINT8(C), ?IS_UINT8(D) -> ipv4;
|
||||||
|
ip_version({A, B, C, D, E, F, G, H}) when ?IS_UINT16(A), ?IS_UINT16(B), ?IS_UINT16(C), ?IS_UINT16(D),
|
||||||
|
?IS_UINT16(E), ?IS_UINT16(F), ?IS_UINT16(G), ?IS_UINT16(H) -> ipv6;
|
||||||
|
ip_version(Ip) when is_binary(Ip) ->
|
||||||
|
ip_version(binary_to_list(Ip));
|
||||||
|
ip_version(Ip) when is_list(Ip) ->
|
||||||
case inet_parse:address(Ip) of
|
case inet_parse:address(Ip) of
|
||||||
{ok, Addr} ->
|
{ok, {_, _, _, _}} -> ipv4;
|
||||||
ipv4_to_n(Addr);
|
{ok, {_, _, _, _, _, _, _, _}} -> ipv6;
|
||||||
|
_ -> {error, bad_ip_format}
|
||||||
|
end;
|
||||||
|
ip_version(_) ->
|
||||||
|
{error, bad_ip_format}.
|
||||||
|
|
||||||
|
-spec ip_to_bytes(Ip :: tuple() | list() | binary() | integer()) ->
|
||||||
|
{ok, ipv4 | ipv6, binary()} | {error, atom()}.
|
||||||
|
ip_to_bytes(Ip) when is_integer(Ip), Ip >= 0, Ip =< 16#FFFFFFFF ->
|
||||||
|
{ok, ipv4, <<Ip:32>>};
|
||||||
|
ip_to_bytes({A, B, C, D}) when ?IS_UINT8(A), ?IS_UINT8(B), ?IS_UINT8(C), ?IS_UINT8(D) ->
|
||||||
|
{ok, ipv4, <<A, B, C, D>>};
|
||||||
|
ip_to_bytes({A, B, C, D, E, F, G, H}) when ?IS_UINT16(A), ?IS_UINT16(B), ?IS_UINT16(C), ?IS_UINT16(D),
|
||||||
|
?IS_UINT16(E), ?IS_UINT16(F), ?IS_UINT16(G), ?IS_UINT16(H) ->
|
||||||
|
{ok, ipv6, <<A:16, B:16, C:16, D:16, E:16, F:16, G:16, H:16>>};
|
||||||
|
ip_to_bytes(Ip) when is_binary(Ip) ->
|
||||||
|
ip_to_bytes(binary_to_list(Ip));
|
||||||
|
ip_to_bytes(Ip) when is_list(Ip) ->
|
||||||
|
case inet_parse:address(Ip) of
|
||||||
|
{ok, {A, B, C, D}} ->
|
||||||
|
{ok, ipv4, <<A, B, C, D>>};
|
||||||
|
{ok, {A, B, C, D, E, F, G, H}} ->
|
||||||
|
{ok, ipv6, <<A:16, B:16, C:16, D:16, E:16, F:16, G:16, H:16>>};
|
||||||
_ ->
|
_ ->
|
||||||
{error, bad_ip_format}
|
{error, bad_ip_format}
|
||||||
end.
|
end;
|
||||||
|
ip_to_bytes(_) ->
|
||||||
|
{error, bad_ip_format}.
|
||||||
|
|
||||||
|
-spec ipv4_to_n(Ip :: tuple() | list() | binary() | integer()) ->
|
||||||
|
non_neg_integer() | {error, atom()}.
|
||||||
|
ipv4_to_n(IntIp) when is_integer(IntIp), IntIp >= 0, IntIp =< 16#FFFFFFFF -> IntIp;
|
||||||
|
ipv4_to_n({A, B, C, D}) when ?IS_UINT8(A), ?IS_UINT8(B), ?IS_UINT8(C), ?IS_UINT8(D) ->
|
||||||
|
<<N:32>> = <<A, B, C, D>>,
|
||||||
|
N;
|
||||||
|
ipv4_to_n(Ip) when is_binary(Ip) ->
|
||||||
|
ipv4_to_n(binary_to_list(Ip));
|
||||||
|
ipv4_to_n(Ip) when is_list(Ip) ->
|
||||||
|
case ip_to_bytes(Ip) of
|
||||||
|
{ok, ipv4, <<N:32>>} -> N;
|
||||||
|
_ -> {error, bad_ip_format}
|
||||||
|
end;
|
||||||
|
ipv4_to_n(_) ->
|
||||||
|
{error, bad_ip_format}.
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
%% license that can be found in the LICENSE file.
|
%% license that can be found in the LICENSE file.
|
||||||
%%
|
%%
|
||||||
%% @doc
|
%% @doc
|
||||||
%% ip2region xdb client worker
|
%% ip2region xdb client worker, now version-aware (IPv4/IPv6).
|
||||||
%% @end
|
%% @end
|
||||||
%%%-------------------------------------------------------------------
|
%%%-------------------------------------------------------------------
|
||||||
-module(ip2region_worker).
|
-module(ip2region_worker).
|
||||||
|
|
@ -15,7 +15,11 @@
|
||||||
-export([start/1, stop/1, start_link/1]).
|
-export([start/1, stop/1, start_link/1]).
|
||||||
-export([search/2]).
|
-export([search/2]).
|
||||||
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
|
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
|
||||||
-record(state, {xdb_fd}).
|
|
||||||
|
-record(state, {
|
||||||
|
xdb_fd,
|
||||||
|
version :: ipv4 | ipv6
|
||||||
|
}).
|
||||||
|
|
||||||
%%==========================================
|
%%==========================================
|
||||||
%% API
|
%% API
|
||||||
|
|
@ -28,7 +32,6 @@ start_link(Args) ->
|
||||||
Opts = [{spawn_opt, [{min_heap_size, 6000}]}],
|
Opts = [{spawn_opt, [{min_heap_size, 6000}]}],
|
||||||
gen_server:start_link(?MODULE, Args, Opts).
|
gen_server:start_link(?MODULE, Args, Opts).
|
||||||
|
|
||||||
|
|
||||||
stop(Pid) ->
|
stop(Pid) ->
|
||||||
gen_server:call(Pid, stop).
|
gen_server:call(Pid, stop).
|
||||||
|
|
||||||
|
|
@ -38,7 +41,7 @@ search(Pid, Ip) ->
|
||||||
%%==========================================
|
%%==========================================
|
||||||
%% gen_server callbacks
|
%% gen_server callbacks
|
||||||
%% =========================================
|
%% =========================================
|
||||||
init(_Args) ->
|
init(Args) ->
|
||||||
process_flag(trap_exit, true),
|
process_flag(trap_exit, true),
|
||||||
AppName =
|
AppName =
|
||||||
case application:get_application() of
|
case application:get_application() of
|
||||||
|
|
@ -46,11 +49,31 @@ init(_Args) ->
|
||||||
_ -> ?APP_NAME
|
_ -> ?APP_NAME
|
||||||
end,
|
end,
|
||||||
PrivDir = code:priv_dir(AppName),
|
PrivDir = code:priv_dir(AppName),
|
||||||
XdbFileName = filename:join([PrivDir, "ip2region.xdb"]),
|
XdbFileName =
|
||||||
|
case proplists:get_value(xdb_file, Args) of
|
||||||
|
undefined -> filename:join([PrivDir, "ip2region.xdb"]);
|
||||||
|
Path ->
|
||||||
|
case filename:pathtype(Path) of
|
||||||
|
absolute -> Path;
|
||||||
|
_ -> filename:join([PrivDir, Path])
|
||||||
|
end
|
||||||
|
end,
|
||||||
error_logger:info_report(io_lib:format("XdbFile:~s~n", [XdbFileName])),
|
error_logger:info_report(io_lib:format("XdbFile:~s~n", [XdbFileName])),
|
||||||
{ok, IoDevice} = file:open(XdbFileName, [read, binary]),
|
{ok, IoDevice} = file:open(XdbFileName, [read, binary]),
|
||||||
load_vector_index(IoDevice),
|
{ok, HeaderBin} = file:read(IoDevice, ?XDB_HEADER_SIZE),
|
||||||
{ok, #state{xdb_fd = IoDevice}}.
|
{ok, Header} = ip2region_xdb:parse_header(HeaderBin),
|
||||||
|
ExpectedVersion = proplists:get_value(expected_version, Args),
|
||||||
|
case resolve_version(Header) of
|
||||||
|
{ok, Version} when ExpectedVersion =:= undefined; ExpectedVersion =:= Version ->
|
||||||
|
load_vector_index(IoDevice, Version),
|
||||||
|
{ok, #state{xdb_fd = IoDevice, version = Version}};
|
||||||
|
{ok, Version} ->
|
||||||
|
file:close(IoDevice),
|
||||||
|
{stop, {xdb_version_mismatch, ExpectedVersion, Version}};
|
||||||
|
{error, Reason} ->
|
||||||
|
file:close(IoDevice),
|
||||||
|
{stop, Reason}
|
||||||
|
end.
|
||||||
|
|
||||||
handle_call(Request, From, State) ->
|
handle_call(Request, From, State) ->
|
||||||
try
|
try
|
||||||
|
|
@ -85,22 +108,19 @@ handle_info(Info, State) ->
|
||||||
terminate(_Reason, State) ->
|
terminate(_Reason, State) ->
|
||||||
#state{xdb_fd = XdbFd} = State,
|
#state{xdb_fd = XdbFd} = State,
|
||||||
case is_pid(XdbFd) of
|
case is_pid(XdbFd) of
|
||||||
true ->
|
true -> file:close(XdbFd);
|
||||||
file:close(XdbFd);
|
_ -> skip
|
||||||
_ ->
|
|
||||||
skip
|
|
||||||
end,
|
end,
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
code_change(_OldVsn, State, _Extra) ->
|
code_change(_OldVsn, State, _Extra) ->
|
||||||
{ok, State}.
|
{ok, State}.
|
||||||
|
|
||||||
|
|
||||||
%%==========================================
|
%%==========================================
|
||||||
%% Internal function
|
%% Internal function
|
||||||
%% =========================================
|
%% =========================================
|
||||||
do_call({search, Ip}, _From, #state{xdb_fd = IoDevice} = State) ->
|
do_call({search, Ip}, _From, #state{xdb_fd = IoDevice} = State) ->
|
||||||
Reply = search_ip(IoDevice, Ip),
|
Reply = search_ip(IoDevice, Ip, State),
|
||||||
{reply, Reply, State};
|
{reply, Reply, State};
|
||||||
|
|
||||||
do_call(stop, _From, State) ->
|
do_call(stop, _From, State) ->
|
||||||
|
|
@ -108,7 +128,7 @@ do_call(stop, _From, State) ->
|
||||||
|
|
||||||
do_call(Request, From, State) ->
|
do_call(Request, From, State) ->
|
||||||
error_logger:error_report(io_lib:format("unknown request: ~p, from:~p", [Request, From])),
|
error_logger:error_report(io_lib:format("unknown request: ~p, from:~p", [Request, From])),
|
||||||
{noreply, State}.
|
{reply, {error, unknown_request}, State}.
|
||||||
|
|
||||||
do_cast(Msg, State) ->
|
do_cast(Msg, State) ->
|
||||||
error_logger:error_report(io_lib:format("unknown msg: ~p", [Msg])),
|
error_logger:error_report(io_lib:format("unknown msg: ~p", [Msg])),
|
||||||
|
|
@ -118,66 +138,104 @@ do_info(Info, State) ->
|
||||||
error_logger:error_report(io:format("unknown info: ~p", [Info])),
|
error_logger:error_report(io:format("unknown info: ~p", [Info])),
|
||||||
{noreply, State}.
|
{noreply, State}.
|
||||||
|
|
||||||
|
-spec resolve_version(#xdb_header{}) -> {ok, ipv4 | ipv6} | {error, term()}.
|
||||||
load_vector_index(IoDevice) ->
|
resolve_version(Header) ->
|
||||||
Key = ip2region_header_loaded,
|
case ip2region_xdb:header_version(Header) of
|
||||||
case persistent_term:get(Key, false) of
|
2 -> {ok, ipv4};
|
||||||
true -> ok;
|
3 ->
|
||||||
_ ->
|
case ip2region_xdb:header_ip_version(Header) of
|
||||||
{ok, <<_Header:?XDB_HEADER_SIZE/binary, VectorIndexBin/binary>> } =
|
?IP_VERSION_4 -> {ok, ipv4};
|
||||||
file:read(IoDevice, ?XDB_HEADER_SIZE + ?XDB_VECTOR_INDEX_COUNT*8),
|
?IP_VERSION_6 -> {ok, ipv6};
|
||||||
load_vector_index_aux(VectorIndexBin, 0),
|
V -> {error, {invalid_xdb_ip_version, V}}
|
||||||
persistent_term:put(Key, true)
|
end;
|
||||||
|
V -> {error, {invalid_xdb_version, V}}
|
||||||
end.
|
end.
|
||||||
|
|
||||||
load_vector_index_aux(<<>>, _Index) -> ok;
|
load_vector_index(IoDevice, Version) ->
|
||||||
load_vector_index_aux(<<SPtr:32/little, EPtr:32/little, VectorIndexBin/binary>>, Index) ->
|
%% The vector index ETS table is global and shared by all pool workers.
|
||||||
Term = {Index, SPtr, EPtr},
|
%% Only the first worker (or the first worker after a restart) populates it.
|
||||||
ets:insert(?XDB_VECTOR_INDEX, Term),
|
%% Subsequent workers skip the file:read here; search_ip/2 uses file:pread/3,
|
||||||
load_vector_index_aux(VectorIndexBin, Index + 1).
|
%% so the current file pointer position does not affect correctness.
|
||||||
|
Table = ip2region_xdb:vector_index_table(Version),
|
||||||
|
case ets:info(Table, size) of
|
||||||
search_ip(IoDevice, Ip) ->
|
?XDB_VECTOR_INDEX_COUNT ->
|
||||||
IntIp = ip2region_util:ipv4_to_n(Ip),
|
ok;
|
||||||
case ets:lookup(?IP2REGION_CACHE, IntIp) of
|
undefined ->
|
||||||
[{_IntIp, RegionInfo}] ->
|
Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}],
|
||||||
RegionInfo;
|
ets:new(Table, Opts),
|
||||||
|
load_vector_index_data(IoDevice, Table);
|
||||||
_ ->
|
_ ->
|
||||||
<<A:8, B:8, _Rest/binary>> = <<IntIp:32>>,
|
%% A previous worker may have crashed midway; reload to be safe.
|
||||||
VectorIdx = A * ?XDB_VECTOR_COLS + B,
|
load_vector_index_data(IoDevice, Table)
|
||||||
[{_, SPtr, EPtr}] = ets:lookup(?XDB_VECTOR_INDEX, VectorIdx),
|
|
||||||
RegionInfo = search_ip(IoDevice, IntIp, SPtr, EPtr, 0, (EPtr - SPtr) div ?XDB_SEGMENT_INDEX_SIZE),
|
|
||||||
ets:insert_new(?IP2REGION_CACHE, {IntIp, RegionInfo}),
|
|
||||||
RegionInfo
|
|
||||||
end.
|
end.
|
||||||
|
|
||||||
search_ip(IoDevice, IntIp, SPtr, EPtr, Low, High) when Low =< High ->
|
load_vector_index_data(IoDevice, Table) ->
|
||||||
|
{ok, VectorIndexBin} =
|
||||||
|
file:read(IoDevice, ?XDB_VECTOR_INDEX_COUNT * ?XDB_VECTOR_INDEX_SIZE),
|
||||||
|
load_vector_index_aux(VectorIndexBin, 0, Table).
|
||||||
|
|
||||||
|
load_vector_index_aux(<<>>, _Index, _Table) -> ok;
|
||||||
|
load_vector_index_aux(<<SPtr:32/little, EPtr:32/little, VectorIndexBin/binary>>, Index, Table) ->
|
||||||
|
ets:insert(Table, {Index, SPtr, EPtr}),
|
||||||
|
load_vector_index_aux(VectorIndexBin, Index + 1, Table).
|
||||||
|
|
||||||
|
search_ip(IoDevice, Ip, #state{version = Version}) ->
|
||||||
|
SegSize = ip2region_xdb:segment_index_size(Version),
|
||||||
|
VectorTable = ip2region_xdb:vector_index_table(Version),
|
||||||
|
SegmentTable = ip2region_xdb:segment_index_table(Version),
|
||||||
|
<<A:8, B:8, _/binary>> = Ip,
|
||||||
|
VectorIdx = A * ?XDB_VECTOR_COLS + B,
|
||||||
|
[{_, SPtr, EPtr}] = ets:lookup(VectorTable, VectorIdx),
|
||||||
|
search_ip(IoDevice, Ip, SPtr, EPtr, 0,
|
||||||
|
(EPtr - SPtr) div SegSize, SegSize, Version, SegmentTable).
|
||||||
|
|
||||||
|
search_ip(IoDevice, Ip, SPtr, EPtr, Low, High, SegSize, Version, SegmentTable) when Low =< High ->
|
||||||
Middle = (Low + High) bsr 1,
|
Middle = (Low + High) bsr 1,
|
||||||
SPtr2 = SPtr + Middle * ?XDB_SEGMENT_INDEX_SIZE,
|
SPtr2 = SPtr + Middle * SegSize,
|
||||||
{SIp, EIp, DataLen, DataPtr} = read_segement_index(IoDevice, SPtr2),
|
{SIp, EIp, DataLen, DataPtr} = read_segment_index(IoDevice, SPtr2, SegSize, SegmentTable),
|
||||||
if
|
case ip_in_range(Ip, SIp, EIp, Version) of
|
||||||
IntIp < SIp ->
|
below ->
|
||||||
search_ip(IoDevice, IntIp, SPtr, EPtr, Low, Middle - 1);
|
search_ip(IoDevice, Ip, SPtr, EPtr, Low, Middle - 1, SegSize, Version, SegmentTable);
|
||||||
IntIp > EIp ->
|
above ->
|
||||||
search_ip(IoDevice, IntIp, SPtr, EPtr, Middle + 1, High);
|
search_ip(IoDevice, Ip, SPtr, EPtr, Middle + 1, High, SegSize, Version, SegmentTable);
|
||||||
true ->
|
inside ->
|
||||||
{ok, DataBin} = read_file(IoDevice, DataPtr, DataLen),
|
{ok, DataBin} = read_file(IoDevice, DataPtr, DataLen),
|
||||||
unicode:characters_to_nfc_list(DataBin)
|
unicode:characters_to_nfc_list(DataBin)
|
||||||
end;
|
end;
|
||||||
search_ip(_IoDevice, _IntIp, _SPtr, _EPtr, _Low, _High) ->
|
search_ip(_IoDevice, _Ip, _SPtr, _EPtr, _Low, _High, _SegSize, _Version, _SegmentTable) ->
|
||||||
{error, unknown}.
|
{error, unknown}.
|
||||||
|
|
||||||
read_file(IoDevice, Position, DataLength) ->
|
ip_in_range(Ip, SIp, EIp, ipv4) ->
|
||||||
file:position(IoDevice, {bof, Position}),
|
<<InputInt:32>> = Ip,
|
||||||
file:read(IoDevice, DataLength).
|
if
|
||||||
|
InputInt < SIp -> below;
|
||||||
|
InputInt > EIp -> above;
|
||||||
|
true -> inside
|
||||||
|
end;
|
||||||
|
ip_in_range(Ip, SIp, EIp, ipv6) ->
|
||||||
|
if
|
||||||
|
Ip < SIp -> below;
|
||||||
|
Ip > EIp -> above;
|
||||||
|
true -> inside
|
||||||
|
end.
|
||||||
|
|
||||||
read_segement_index(IoDevice, SPtr) ->
|
read_file(IoDevice, Position, DataLength) ->
|
||||||
case ets:lookup(?XDB_SEGMENT_INDEX, SPtr) of
|
file:pread(IoDevice, Position, DataLength).
|
||||||
|
|
||||||
|
read_segment_index(IoDevice, SPtr, SegSize, SegmentTable) ->
|
||||||
|
case ets:lookup(SegmentTable, SPtr) of
|
||||||
[{_SPtr, SIp, EIp, DataLen, DataPtr}] ->
|
[{_SPtr, SIp, EIp, DataLen, DataPtr}] ->
|
||||||
{SIp, EIp, DataLen, DataPtr};
|
{SIp, EIp, DataLen, DataPtr};
|
||||||
_ ->
|
_ ->
|
||||||
{ok, <<SIp:32/little, EIp:32/little, DataLen:16/little, DataPtr:32/little>>} =
|
{ok, Bin} = read_file(IoDevice, SPtr, SegSize),
|
||||||
read_file(IoDevice, SPtr, ?XDB_SEGMENT_INDEX_SIZE),
|
{SIp, EIp, DataLen, DataPtr} = decode_segment_index(Bin, SegSize),
|
||||||
ets:insert_new(?XDB_SEGMENT_INDEX, {SPtr, SIp, EIp, DataLen, DataPtr}),
|
ets:insert_new(SegmentTable, {SPtr, SIp, EIp, DataLen, DataPtr}),
|
||||||
{SIp, EIp, DataLen, DataPtr}
|
{SIp, EIp, DataLen, DataPtr}
|
||||||
end.
|
end.
|
||||||
|
|
||||||
|
decode_segment_index(Bin, ?XDB_SEGMENT_INDEX_SIZE_V4) ->
|
||||||
|
<<SIp:32/little, EIp:32/little, DataLen:16/little, DataPtr:32/little>> = Bin,
|
||||||
|
{SIp, EIp, DataLen, DataPtr};
|
||||||
|
decode_segment_index(Bin, ?XDB_SEGMENT_INDEX_SIZE_V6) ->
|
||||||
|
<<SIp:16/binary, EIp:16/binary, DataLen:16/little, DataPtr:32/little>> = Bin,
|
||||||
|
{SIp, EIp, DataLen, DataPtr}.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
%%%-------------------------------------------------------------------
|
||||||
|
%% Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
|
%% Use of this source code is governed by a Apache2.0-style
|
||||||
|
%% license that can be found in the LICENSE file.
|
||||||
|
%%
|
||||||
|
%% @doc
|
||||||
|
%% ip2region xdb binary format helpers.
|
||||||
|
%% @end
|
||||||
|
%%%-------------------------------------------------------------------
|
||||||
|
-module(ip2region_xdb).
|
||||||
|
|
||||||
|
-export([parse_header/1]).
|
||||||
|
-export([header_version/1, header_index_policy/1, header_created_at/1,
|
||||||
|
header_start_index_ptr/1, header_end_index_ptr/1,
|
||||||
|
header_ip_version/1, header_runtime_ptr_bytes/1]).
|
||||||
|
-export([segment_index_size/1]).
|
||||||
|
-export([vector_index_table/1, segment_index_table/1, cache_table/1]).
|
||||||
|
|
||||||
|
-include("ip2region.hrl").
|
||||||
|
|
||||||
|
-spec parse_header(binary()) -> {ok, #xdb_header{}} | {error, invalid_header}.
|
||||||
|
parse_header(Bin) when is_binary(Bin), byte_size(Bin) >= 20 ->
|
||||||
|
<<Version:16/little, IndexPolicy:16/little, CreatedAt:32/little,
|
||||||
|
StartIndexPtr:32/little, EndIndexPtr:32/little,
|
||||||
|
IpVersion:16/little, RuntimePtrBytes:16/little, _/binary>> = Bin,
|
||||||
|
{ok, #xdb_header{
|
||||||
|
version = Version,
|
||||||
|
index_policy = IndexPolicy,
|
||||||
|
created_at = CreatedAt,
|
||||||
|
start_index_ptr = StartIndexPtr,
|
||||||
|
end_index_ptr = EndIndexPtr,
|
||||||
|
ip_version = IpVersion,
|
||||||
|
runtime_ptr_bytes = RuntimePtrBytes
|
||||||
|
}};
|
||||||
|
parse_header(_) ->
|
||||||
|
{error, invalid_header}.
|
||||||
|
|
||||||
|
header_version(#xdb_header{version = V}) -> V.
|
||||||
|
header_index_policy(#xdb_header{index_policy = P}) -> P.
|
||||||
|
header_created_at(#xdb_header{created_at = T}) -> T.
|
||||||
|
header_start_index_ptr(#xdb_header{start_index_ptr = P}) -> P.
|
||||||
|
header_end_index_ptr(#xdb_header{end_index_ptr = P}) -> P.
|
||||||
|
header_ip_version(#xdb_header{ip_version = V}) -> V.
|
||||||
|
header_runtime_ptr_bytes(#xdb_header{runtime_ptr_bytes = B}) -> B.
|
||||||
|
|
||||||
|
-spec segment_index_size(ipv4 | ipv6) -> pos_integer().
|
||||||
|
segment_index_size(ipv4) -> ?XDB_SEGMENT_INDEX_SIZE_V4;
|
||||||
|
segment_index_size(ipv6) -> ?XDB_SEGMENT_INDEX_SIZE_V6.
|
||||||
|
|
||||||
|
-spec vector_index_table(ipv4 | ipv6) -> atom().
|
||||||
|
vector_index_table(ipv4) -> ?XDB_VECTOR_INDEX_V4;
|
||||||
|
vector_index_table(ipv6) -> ?XDB_VECTOR_INDEX_V6.
|
||||||
|
|
||||||
|
-spec segment_index_table(ipv4 | ipv6) -> atom().
|
||||||
|
segment_index_table(ipv4) -> ?XDB_SEGMENT_INDEX_V4;
|
||||||
|
segment_index_table(ipv6) -> ?XDB_SEGMENT_INDEX_V6.
|
||||||
|
|
||||||
|
-spec cache_table(ipv4 | ipv6) -> atom().
|
||||||
|
cache_table(ipv4) -> ?IP2REGION_CACHE_V4;
|
||||||
|
cache_table(ipv6) -> ?IP2REGION_CACHE_V6.
|
||||||
|
|
@ -1,11 +1,10 @@
|
||||||
|
|
||||||
%%%-------------------------------------------------------------------
|
%%%-------------------------------------------------------------------
|
||||||
%% Copyright 2022 The Ip2Region Authors. All rights reserved.
|
%% Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
%% Use of this source code is governed by a Apache2.0-style
|
%% Use of this source code is governed by a Apache2.0-style
|
||||||
%% license that can be found in the LICENSE file.
|
%% license that can be found in the LICENSE file.
|
||||||
%%
|
%%
|
||||||
%% @doc
|
%% @doc
|
||||||
%% ip2region xdb client search api
|
%% ip2region xdb client search api with IPv4/IPv6 auto-routing.
|
||||||
%% @end
|
%% @end
|
||||||
%%%-------------------------------------------------------------------
|
%%%-------------------------------------------------------------------
|
||||||
-module(xdb).
|
-module(xdb).
|
||||||
|
|
@ -13,20 +12,37 @@
|
||||||
|
|
||||||
-export([search/1]).
|
-export([search/1]).
|
||||||
|
|
||||||
-spec search(Ip :: tuple() | list() | binary()) -> Result :: binary | {error, Reason::atom()}.
|
-spec search(Ip :: tuple() | list() | binary() | integer()) ->
|
||||||
search(Ip) when is_integer(Ip); is_list(Ip); is_tuple(Ip); is_binary(Ip) ->
|
Result :: list() | {error, term()}.
|
||||||
case ip2region_util:ipv4_to_n(Ip) of
|
search(Ip) ->
|
||||||
IntIp when is_integer(IntIp) ->
|
case ip2region_util:ip_to_bytes(Ip) of
|
||||||
case ets:lookup(?IP2REGION_CACHE, IntIp) of
|
{ok, ipv4, IpBin} ->
|
||||||
[{_IntIp, Region}] -> Region;
|
do_search(?IP2REGION_POOL_V4, ipv4, IpBin);
|
||||||
_ ->
|
{ok, ipv6, IpBin} ->
|
||||||
Worker = poolboy:checkout(?IP2REGION_POOL, true, infinity),
|
do_search(?IP2REGION_POOL_V6, ipv6, IpBin);
|
||||||
try
|
Ret ->
|
||||||
ip2region_worker:search(Worker, IntIp)
|
Ret
|
||||||
after
|
end.
|
||||||
poolboy:checkin(?IP2REGION_POOL, Worker)
|
|
||||||
end
|
do_search(PoolName, Version, IpBin) ->
|
||||||
end;
|
CacheTable = ip2region_xdb:cache_table(Version),
|
||||||
Ret ->
|
case ets:lookup(CacheTable, IpBin) of
|
||||||
Ret
|
[{_, Region}] -> Region;
|
||||||
|
_ ->
|
||||||
|
case whereis(PoolName) of
|
||||||
|
undefined ->
|
||||||
|
{error, pool_not_configured};
|
||||||
|
_ ->
|
||||||
|
Worker = poolboy:checkout(PoolName, true, infinity),
|
||||||
|
try
|
||||||
|
case ip2region_worker:search(Worker, IpBin) of
|
||||||
|
{error, _} = Err -> Err;
|
||||||
|
Region ->
|
||||||
|
ets:insert(CacheTable, {IpBin, Region}),
|
||||||
|
Region
|
||||||
|
end
|
||||||
|
after
|
||||||
|
poolboy:checkin(PoolName, Worker)
|
||||||
|
end
|
||||||
|
end
|
||||||
end.
|
end.
|
||||||
|
|
|
||||||
|
|
@ -11,29 +11,84 @@
|
||||||
-export([main/1]).
|
-export([main/1]).
|
||||||
|
|
||||||
main(DataFile) ->
|
main(DataFile) ->
|
||||||
application:ensure_started(ip2region),
|
%% Keep benchmark output clean while still surfacing real errors.
|
||||||
|
_ = logger:set_handler_config(default, level, error),
|
||||||
|
_ = logger:set_primary_config(level, error),
|
||||||
|
{ok, _} = application:ensure_all_started(ip2region),
|
||||||
show_hw_sw_info(),
|
show_hw_sw_info(),
|
||||||
IpList = load_test_data(DataFile),
|
IpList = load_test_data(DataFile),
|
||||||
run(IpList).
|
run(IpList).
|
||||||
|
|
||||||
show_hw_sw_info() ->
|
show_hw_sw_info() ->
|
||||||
io:format("CPU info:~n", []),
|
{Model, Clock, Cores, Threads} = cpu_info(),
|
||||||
io:format("~s", [os:cmd("egrep '^model name' /proc/cpuinfo | head -1")]),
|
io:format("~nSystem:~n", []),
|
||||||
io:format("~s", [os:cmd("egrep '^cache' /proc/cpuinfo | head -1")]),
|
io:format(" CPU : ~s", [Model]),
|
||||||
io:format("~s", [os:cmd("egrep '^cpu MHz' /proc/cpuinfo | head -1")]),
|
case Clock of
|
||||||
io:format("~s", [os:cmd("egrep '^bogomips' /proc/cpuinfo | head -1")]),
|
"" -> io:format("~n", []);
|
||||||
io:format("cores/threads : ~s~n", [os:cmd("egrep -c '^processor' /proc/cpuinfo")]),
|
_ -> io:format(" @ ~s~n", [Clock])
|
||||||
io:format("Erlang info:~n", []),
|
end,
|
||||||
io:format("system_version:~s", [erlang:system_info(system_version)]),
|
io:format(" Cores : ~s cores / ~s threads~n", [Cores, Threads]),
|
||||||
|
io:format(" Erlang : ~s~n", [string:trim(erlang:system_info(system_version))]),
|
||||||
ok.
|
ok.
|
||||||
|
|
||||||
|
cpu_info() ->
|
||||||
|
case os:type() of
|
||||||
|
{unix, darwin} ->
|
||||||
|
Model = sysctl("machdep.cpu.brand_string"),
|
||||||
|
Clock = first_non_empty([
|
||||||
|
format_clock(sysctl("hw.cpufrequency")),
|
||||||
|
format_clock(sysctl("hw.perflevel0.frequency")),
|
||||||
|
format_clock(sysctl("hw.perflevel1.frequency"))
|
||||||
|
]),
|
||||||
|
Cores = sysctl("hw.physicalcpu"),
|
||||||
|
Threads = sysctl("hw.logicalcpu"),
|
||||||
|
{Model, Clock, Cores, Threads};
|
||||||
|
{unix, linux} ->
|
||||||
|
Model = linux_cpu_field("model name"),
|
||||||
|
Clock = format_clock_mhz(linux_cpu_field("cpu MHz")),
|
||||||
|
Cores = string:trim(os:cmd("grep -c '^processor' /proc/cpuinfo 2>/dev/null")),
|
||||||
|
Threads = Cores,
|
||||||
|
{Model, Clock, Cores, Threads};
|
||||||
|
_ ->
|
||||||
|
{"unknown", "", "?", "?"}
|
||||||
|
end.
|
||||||
|
|
||||||
|
sysctl(Key) ->
|
||||||
|
string:trim(os:cmd("sysctl -n " ++ Key ++ " 2>/dev/null")).
|
||||||
|
|
||||||
|
linux_cpu_field(Key) ->
|
||||||
|
Cmd = "grep -m1 '^" ++ Key ++ "' /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//'",
|
||||||
|
string:trim(os:cmd(Cmd)).
|
||||||
|
|
||||||
|
first_non_empty(["" | Rest]) -> first_non_empty(Rest);
|
||||||
|
first_non_empty([Val | _]) -> Val;
|
||||||
|
first_non_empty([]) -> "".
|
||||||
|
|
||||||
|
format_clock(HzStr) ->
|
||||||
|
case string:to_integer(string:trim(HzStr)) of
|
||||||
|
{ok, Hz, _} when Hz > 1000000000 ->
|
||||||
|
lists:flatten(io_lib:format("~.2f GHz", [Hz / 1000000000]));
|
||||||
|
{ok, Hz, _} when Hz > 1000000 ->
|
||||||
|
lists:flatten(io_lib:format("~.2f MHz", [Hz / 1000000]));
|
||||||
|
_ ->
|
||||||
|
""
|
||||||
|
end.
|
||||||
|
|
||||||
|
format_clock_mhz(MhzStr) ->
|
||||||
|
case string:to_float(string:trim(MhzStr)) of
|
||||||
|
{ok, Mhz, _} ->
|
||||||
|
lists:flatten(io_lib:format("~.3f GHz", [Mhz / 1000]));
|
||||||
|
_ ->
|
||||||
|
""
|
||||||
|
end.
|
||||||
|
|
||||||
load_test_data(DataFile) ->
|
load_test_data(DataFile) ->
|
||||||
{ok, Fd} = file:open(DataFile, [read]),
|
{ok, Fd} = file:open(DataFile, [read]),
|
||||||
T0 = os:timestamp(),
|
T0 = os:timestamp(),
|
||||||
IpList = load_test_data(Fd, []),
|
IpList = load_test_data(Fd, []),
|
||||||
T1 = os:timestamp(),
|
T1 = os:timestamp(),
|
||||||
Sec = timer:now_diff(T1, T0) / 1000000,
|
Sec = timer:now_diff(T1, T0) / 1000000,
|
||||||
io:format("load test data use ~ps~n", [Sec]),
|
io:format(" Loaded : ~p IPs in ~.3f s~n", [length(IpList), Sec]),
|
||||||
IpList.
|
IpList.
|
||||||
|
|
||||||
load_test_data(Fd, IpList) ->
|
load_test_data(Fd, IpList) ->
|
||||||
|
|
@ -46,30 +101,30 @@ load_test_data(Fd, IpList) ->
|
||||||
load_test_data(Fd, IpList)
|
load_test_data(Fd, IpList)
|
||||||
end;
|
end;
|
||||||
_ ->
|
_ ->
|
||||||
file:close(Fd),
|
file:close(Fd),
|
||||||
IpList
|
IpList
|
||||||
end.
|
end.
|
||||||
|
|
||||||
run(IpList) ->
|
run(IpList) ->
|
||||||
garbage_collect(),
|
garbage_collect(),
|
||||||
io:format("~nstart run benchmark tests~n", []),
|
io:format("~nBenchmarks:~n", []),
|
||||||
io:format("~nsearch from file:~n", []),
|
run_test("cold", IpList),
|
||||||
run_test(IpList),
|
run_test("warm", IpList),
|
||||||
io:format("~nsearch from cache:~n", []),
|
io:format("~nDone.~n", []).
|
||||||
run_test(IpList),
|
|
||||||
io:format("~nbenchmark test finish~n", []).
|
|
||||||
|
|
||||||
run_test(IpList) ->
|
run_test(Label, IpList) ->
|
||||||
T0 = os:timestamp(),
|
T0 = os:timestamp(),
|
||||||
run_test_aux(IpList),
|
run_test_aux(IpList),
|
||||||
T1 = os:timestamp(),
|
T1 = os:timestamp(),
|
||||||
Sec = timer:now_diff(T1, T0) / 1000000,
|
Sec = timer:now_diff(T1, T0) / 1000000,
|
||||||
IpCount = length(IpList),
|
Count = length(IpList),
|
||||||
io:format("ip count:~p,~ntotal time: ~ps,~nsearch ~p times per second,~nuse ~p micro second per search~n",
|
Qps = Count / Sec,
|
||||||
[IpCount, Sec, IpCount / Sec, Sec * 1000000/IpCount]).
|
MsOp = Sec * 1000 / Count,
|
||||||
|
UsOp = Sec * 1000000 / Count,
|
||||||
|
io:format(" ~-8s total=~7.3fs count=~7w qps=~12.2f avg=~9.6f ms/op (~6.3f us/op)~n",
|
||||||
|
[Label, Sec, Count, Qps, MsOp, UsOp]).
|
||||||
|
|
||||||
run_test_aux([]) -> ok;
|
run_test_aux([]) -> ok;
|
||||||
run_test_aux([Ip | Tail]) ->
|
run_test_aux([Ip | Tail]) ->
|
||||||
xdb:search(Ip),
|
xdb:search(Ip),
|
||||||
run_test_aux(Tail).
|
run_test_aux(Tail).
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
-module(ip2region_sup_test).
|
||||||
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
-include("ip2region.hrl").
|
||||||
|
|
||||||
|
pools_started_test() ->
|
||||||
|
{ok, _} = ip2region_test_util:start_app(),
|
||||||
|
?assert(is_pid(whereis(?IP2REGION_POOL_V4))),
|
||||||
|
?assert(is_pid(whereis(?IP2REGION_POOL_V6))).
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
-module(ip2region_test_util).
|
||||||
|
|
||||||
|
-export([
|
||||||
|
repo_root/0,
|
||||||
|
v6_xdb_path/0,
|
||||||
|
default_db_config/0,
|
||||||
|
reset_app/0,
|
||||||
|
load_app/0,
|
||||||
|
set_db_config/1,
|
||||||
|
start_app/0,
|
||||||
|
start_app/1,
|
||||||
|
stop_app/0
|
||||||
|
]).
|
||||||
|
|
||||||
|
repo_root() ->
|
||||||
|
TestDir = filename:dirname(?FILE),
|
||||||
|
ErlangDir = filename:dirname(TestDir),
|
||||||
|
BindingDir = filename:dirname(ErlangDir),
|
||||||
|
filename:dirname(BindingDir).
|
||||||
|
|
||||||
|
v6_xdb_path() ->
|
||||||
|
filename:join([repo_root(), "data", "ip2region_v6.xdb"]).
|
||||||
|
|
||||||
|
default_db_config() ->
|
||||||
|
[
|
||||||
|
{ipv4, "ip2region.xdb"},
|
||||||
|
{ipv6, v6_xdb_path()}
|
||||||
|
].
|
||||||
|
|
||||||
|
reset_app() ->
|
||||||
|
try application:stop(ip2region) catch _:_ -> ok end,
|
||||||
|
try application:unload(ip2region) catch _:_ -> ok end,
|
||||||
|
ok.
|
||||||
|
|
||||||
|
load_app() ->
|
||||||
|
ok = application:load(ip2region).
|
||||||
|
|
||||||
|
set_db_config(Config) ->
|
||||||
|
ok = application:set_env(ip2region, db, Config).
|
||||||
|
|
||||||
|
start_app() ->
|
||||||
|
start_app(default_db_config()).
|
||||||
|
|
||||||
|
start_app(Config) ->
|
||||||
|
reset_app(),
|
||||||
|
load_app(),
|
||||||
|
set_db_config(Config),
|
||||||
|
application:ensure_all_started(ip2region).
|
||||||
|
|
||||||
|
stop_app() ->
|
||||||
|
application:stop(ip2region).
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
-module(ip2region_util_test).
|
||||||
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
|
||||||
|
ipv4_inputs_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes("1.0.8.0")),
|
||||||
|
?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes(<<"1.0.8.0">>)),
|
||||||
|
?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes({1,0,8,0})),
|
||||||
|
?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes(16779264)),
|
||||||
|
?_assertEqual(ipv4, ip2region_util:ip_version("1.0.8.0")),
|
||||||
|
?_assertEqual(ipv4, ip2region_util:ip_version({1,0,8,0}))
|
||||||
|
].
|
||||||
|
|
||||||
|
ipv6_inputs_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>>},
|
||||||
|
ip2region_util:ip_to_bytes("::1")),
|
||||||
|
?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,255,255,1,0,8,0>>},
|
||||||
|
ip2region_util:ip_to_bytes("::ffff:1.0.8.0")),
|
||||||
|
?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,255,255,1,0,8,0>>},
|
||||||
|
ip2region_util:ip_to_bytes(<<"::ffff:1.0.8.0">>)),
|
||||||
|
?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,255,255,1,0,8,0>>},
|
||||||
|
ip2region_util:ip_to_bytes({0,0,0,0,0,65535,256,2048})),
|
||||||
|
?_assertEqual(ipv6, ip2region_util:ip_version("::1")),
|
||||||
|
?_assertEqual(ipv6, ip2region_util:ip_version({0,0,0,0,0,0,0,1}))
|
||||||
|
].
|
||||||
|
|
||||||
|
invalid_inputs_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes("xxx.0.8.0")),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes("::ggg")),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({1,2,3})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({1,2,3,4,5,6,7})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes(atom)),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes(-1)),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes(16#100000000)),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({256, 0, 0, 1})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({-1, 0, 0, 0})),
|
||||||
|
?_assertEqual({error, bad_ip_format},
|
||||||
|
ip2region_util:ip_to_bytes({0, 0, 0, 0, 0, 0, 0, 65536})),
|
||||||
|
?_assertEqual({error, bad_ip_format},
|
||||||
|
ip2region_util:ip_to_bytes({0, 0, 0, 0, 0, 0, 0, -1})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ipv4_to_n(-1)),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ipv4_to_n(16#100000000)),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ipv4_to_n({256, 0, 0, 1}))
|
||||||
|
].
|
||||||
|
|
||||||
|
ip_version_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual(ipv4, ip2region_util:ip_version(<<"1.0.8.0">>)),
|
||||||
|
?_assertEqual(ipv6, ip2region_util:ip_version(<<"::1">>)),
|
||||||
|
?_assertEqual(ipv4, ip2region_util:ip_version(16779264 + 1)),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version("::ggg")),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version({1,2,3})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version({256,0,0,1})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version({0,0,0,0,0,0,0,65536})),
|
||||||
|
?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version(16#100000000))
|
||||||
|
].
|
||||||
|
|
||||||
|
legacy_ipv4_to_n_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual(16779264, ip2region_util:ipv4_to_n("1.0.8.0")),
|
||||||
|
?_assertEqual(16779264, ip2region_util:ipv4_to_n(<<"1.0.8.0">>)),
|
||||||
|
?_assertEqual(16779264, ip2region_util:ipv4_to_n({1,0,8,0})),
|
||||||
|
?_assertEqual(16779264, ip2region_util:ipv4_to_n(16779264))
|
||||||
|
].
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
-module(ip2region_worker_test).
|
||||||
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
-include("ip2region.hrl").
|
||||||
|
|
||||||
|
worker_search_v4_binary_test() ->
|
||||||
|
ip2region_sup:create_table(),
|
||||||
|
{ok, Pid} = ip2region_worker:start_link([{xdb_file, "ip2region.xdb"}]),
|
||||||
|
try
|
||||||
|
Region = ip2region_worker:search(Pid, <<1,0,8,0>>),
|
||||||
|
?assert(is_list(Region))
|
||||||
|
after
|
||||||
|
ip2region_worker:stop(Pid)
|
||||||
|
end.
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
-module(ip2region_xdb_test).
|
||||||
|
|
||||||
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
-include("ip2region.hrl").
|
||||||
|
|
||||||
|
valid_v4_header_test() ->
|
||||||
|
HeaderBin = <<2:16/little, 1:16/little, 0:32/little,
|
||||||
|
256:32/little, 1024:32/little,
|
||||||
|
?IP_VERSION_4:16/little, 4:16/little>>,
|
||||||
|
{ok, Header} = ip2region_xdb:parse_header(HeaderBin),
|
||||||
|
?assertEqual(2, ip2region_xdb:header_version(Header)),
|
||||||
|
?assertEqual(1, ip2region_xdb:header_index_policy(Header)),
|
||||||
|
?assertEqual(256, ip2region_xdb:header_start_index_ptr(Header)),
|
||||||
|
?assertEqual(1024, ip2region_xdb:header_end_index_ptr(Header)),
|
||||||
|
?assertEqual(?IP_VERSION_4, ip2region_xdb:header_ip_version(Header)),
|
||||||
|
?assertEqual(4, ip2region_xdb:header_runtime_ptr_bytes(Header)).
|
||||||
|
|
||||||
|
valid_v6_header_test() ->
|
||||||
|
HeaderBin = <<3:16/little, 1:16/little, 0:32/little,
|
||||||
|
256:32/little, 1024:32/little,
|
||||||
|
?IP_VERSION_6:16/little, 4:16/little>>,
|
||||||
|
{ok, Header} = ip2region_xdb:parse_header(HeaderBin),
|
||||||
|
?assertEqual(3, ip2region_xdb:header_version(Header)),
|
||||||
|
?assertEqual(?IP_VERSION_6, ip2region_xdb:header_ip_version(Header)).
|
||||||
|
|
||||||
|
invalid_header_test() ->
|
||||||
|
?assertEqual({error, invalid_header}, ip2region_xdb:parse_header(<<0:128>>)).
|
||||||
|
|
||||||
|
table_helpers_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual(?XDB_VECTOR_INDEX_V4, ip2region_xdb:vector_index_table(ipv4)),
|
||||||
|
?_assertEqual(?XDB_VECTOR_INDEX_V6, ip2region_xdb:vector_index_table(ipv6)),
|
||||||
|
?_assertEqual(?XDB_SEGMENT_INDEX_V4, ip2region_xdb:segment_index_table(ipv4)),
|
||||||
|
?_assertEqual(?XDB_SEGMENT_INDEX_V6, ip2region_xdb:segment_index_table(ipv6)),
|
||||||
|
?_assertEqual(?IP2REGION_CACHE_V4, ip2region_xdb:cache_table(ipv4)),
|
||||||
|
?_assertEqual(?IP2REGION_CACHE_V6, ip2region_xdb:cache_table(ipv6))
|
||||||
|
].
|
||||||
|
|
||||||
|
segment_index_size_test_() ->
|
||||||
|
[
|
||||||
|
?_assertEqual(14, ip2region_xdb:segment_index_size(ipv4)),
|
||||||
|
?_assertEqual(38, ip2region_xdb:segment_index_size(ipv6))
|
||||||
|
].
|
||||||
|
|
@ -2,9 +2,11 @@
|
||||||
|
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
|
|
||||||
|
-define(IPV6_RESULT, "United States|Florida|Miami|Google LLC|US").
|
||||||
|
|
||||||
search_test_() ->
|
search_test_() ->
|
||||||
application:ensure_started(ip2region),
|
{ok, _} = ip2region_test_util:start_app(),
|
||||||
A = "中国|0|广东省|广州市|电信",
|
A = "中国|广东省|广州市|中国电信|CN",
|
||||||
Region0 = xdb:search("1.0.8.0"),
|
Region0 = xdb:search("1.0.8.0"),
|
||||||
Region1 = xdb:search(<<"1.0.8.0">>),
|
Region1 = xdb:search(<<"1.0.8.0">>),
|
||||||
Region2 = xdb:search({1,0,8,0}),
|
Region2 = xdb:search({1,0,8,0}),
|
||||||
|
|
@ -14,5 +16,39 @@ search_test_() ->
|
||||||
?_assert(A =:= Region1),
|
?_assert(A =:= Region1),
|
||||||
?_assert(A =:= Region2),
|
?_assert(A =:= Region2),
|
||||||
?_assert({error, bad_ip_format} =:= Region3)
|
?_assert({error, bad_ip_format} =:= Region3)
|
||||||
|
|
||||||
].
|
].
|
||||||
|
|
||||||
|
ipv6_search_test_() ->
|
||||||
|
{ok, _} = ip2region_test_util:start_app(),
|
||||||
|
[
|
||||||
|
?_assert(?IPV6_RESULT =:= xdb:search("2001:4860:4860::8888")),
|
||||||
|
?_assert(?IPV6_RESULT =:= xdb:search(<<"2001:4860:4860::8888">>)),
|
||||||
|
?_assert(?IPV6_RESULT =:= xdb:search({8193, 18528, 18528, 0, 0, 0, 0, 34952}))
|
||||||
|
].
|
||||||
|
|
||||||
|
invalid_search_test_() ->
|
||||||
|
{ok, _} = ip2region_test_util:start_app(),
|
||||||
|
[
|
||||||
|
?_assertEqual({error, bad_ip_format}, xdb:search("xxx.0.8.0")),
|
||||||
|
?_assertEqual({error, bad_ip_format}, xdb:search("::ggg")),
|
||||||
|
?_assertEqual({error, bad_ip_format}, xdb:search({1,2,3}))
|
||||||
|
].
|
||||||
|
|
||||||
|
ipv6_pool_not_configured_test() ->
|
||||||
|
{ok, _} = ip2region_test_util:start_app([{ipv4, "ip2region.xdb"}]),
|
||||||
|
?assertEqual({error, pool_not_configured}, xdb:search("2001:4860:4860::8888")).
|
||||||
|
|
||||||
|
xdb_version_mismatch_test() ->
|
||||||
|
{ok, _} = ip2region_test_util:start_app(),
|
||||||
|
V4File = filename:join([
|
||||||
|
ip2region_test_util:repo_root(), "binding", "erlang", "priv", "ip2region.xdb"
|
||||||
|
]),
|
||||||
|
error_logger:tty(false),
|
||||||
|
try
|
||||||
|
?assertEqual(
|
||||||
|
{error, {xdb_version_mismatch, ipv6, ipv4}},
|
||||||
|
ip2region_worker:start([{xdb_file, V4File}, {expected_version, ipv6}])
|
||||||
|
)
|
||||||
|
after
|
||||||
|
error_logger:tty(true)
|
||||||
|
end.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,225 @@
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
|
||||||
|
==========================================================================
|
||||||
|
The following license applies to the ip2region library
|
||||||
|
--------------------------------------------------------------------------
|
||||||
|
Copyright (c) 2015 Lionsoul<chenxin619315@gmail.com>
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
"Software"), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||||
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||||
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||||
|
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
@ -1,92 +1,103 @@
|
||||||
# ip2region xdb golang 查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
# 使用方式
|
# ip2region golang query client
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### package get
|
||||||
|
|
||||||
### package 获取
|
|
||||||
```bash
|
```bash
|
||||||
go get github.com/lionsoul2014/ip2region/binding/golang
|
go get github.com/lionsoul2014/ip2region/binding/golang
|
||||||
```
|
```
|
||||||
|
|
||||||
### 关于查询服务
|
### About the query service
|
||||||
从 `3.11.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务,**建议优先使用该方式来进行查询调用**,具体使用方式如下:
|
|
||||||
|
Starting from version `3.11.0`, a dual-protocol compatible and concurrency-safe `Ip2Region` query service is provided. **It is recommended to prioritize this method for query calls.** The specific usage is as follows:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
import "github.com/lionsoul2014/ip2region/binding/golang/service"
|
import "github.com/lionsoul2014/ip2region/binding/golang/service"
|
||||||
|
|
||||||
// 1, 创建 v4 的配置:指定缓存策略和 v4 的 xdb 文件路径
|
// 1. Create v4 configuration: specify the cache policy and the v4 xdb file path
|
||||||
// 参数1: 缓存策略, options: service.NoCache / service.VIndexCache / service.BufferCache
|
// Parameter 1: Cache policy, options: service.NoCache / service.VIndexCache / service.BufferCache
|
||||||
// 参数2: xdb 文件路径
|
// Parameter 2: xdb file path
|
||||||
// 参数3: 初始化的查询器数量
|
// Parameter 3: Number of initialized searchers
|
||||||
v4Config, err := service.NewV4Config(service.VIndexCache, "ip2region v4 xdb path", 20)
|
v4Config, err := service.NewV4Config(service.VIndexCache, "ip2region v4 xdb path", 20)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create v4 config: %s", err)
|
return fmt.Errorf("failed to create v4 config: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2, 创建 v6 的配置:指定缓存策略和 v6 的 xdb 文件路径
|
// 2. Create v6 configuration: specify the cache policy and the v6 xdb file path
|
||||||
v6Config, err := service.NewV6Config(service.VIndexCache, "ip2region v6 xdb path", 20)
|
v6Config, err := service.NewV6Config(service.VIndexCache, "ip2region v6 xdb path", 20)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create v6 config: %s", err)
|
return fmt.Errorf("failed to create v6 config: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3,通过上述配置创建 Ip2Region 查询服务
|
// 3. Create the Ip2Region query service using the above configurations
|
||||||
ip2region, err := service.NewIp2Region(v4Config, v6Config)
|
ip2region, err := service.NewIp2Region(v4Config, v6Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create ip2region service: %s", err)
|
return fmt.Errorf("failed to create ip2region service: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4,导出 ip2region 服务进行双版本的IP地址的并发查询,例如:
|
// 4. Export the ip2region service for concurrent dual-version IP address queries, for example:
|
||||||
v4Region, err := ip2region.SearchByStr("113.92.157.29") // 进行 IPv4 查询
|
v4Region, err := ip2region.Search("113.92.157.29") // Perform IPv4 query
|
||||||
v6Region, err := ip2region.SearchByStr("240e:3b7:3272:d8d0:db09:c067:8d59:539e") // 进行 IPv6 查询
|
v6Region, err := ip2region.Search("240e:3b7:3272:d8d0:db09:c067:8d59:539e") // Perform IPv6 query
|
||||||
|
|
||||||
|
|
||||||
// 5,在服务需要关闭的时候,同时关闭 ip2region 查询服务
|
// 5. When the parent service needs to be closed, close the ip2region query service as well
|
||||||
ip2region.Close()
|
ip2region.Close()
|
||||||
```
|
```
|
||||||
|
|
||||||
##### `Ip2Region` 查询备注:
|
##### `Ip2Region` Query Notes:
|
||||||
1. 该查询服务的 API 并发安全且同时支持 IPv4 和 Ipv6 的地址,内部实现会自动判断。
|
|
||||||
2. v4 和 v6 的配置需要单独创建,可以给 v4 和 v6 设置使用不同的缓存策略,也可以指定其中一个为 `nil` 则该版本的 IP 地址查询都会返回 `""`。
|
|
||||||
3. 请结合您的项目的并发数设置一个合适的查询器数量,这个值在运行过程中是固定的,每次查询会从池子里租借一个查询器来完成查询操作,查询完成后再归还回去,如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务。
|
|
||||||
4. 如果配置设置的缓存策略为 `service.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器,该实现天生并发安全,此时指定的查询器数量无效。
|
|
||||||
5. 如果 `Ip2Region` 查询器在提供服务期间,调用 Close 默认会最大等待 10 秒钟来等待尽量多的查询器归还,也可以调用 `CloseTimeout` 来自定义最长等待时间。
|
|
||||||
|
|
||||||
|
1. The API of this query service is concurrency-safe and supports both IPv4 and IPv6 addresses; the internal implementation handles identification automatically.
|
||||||
|
2. v4 and v6 configurations need to be created separately. You can set different cache policies for v4 and v6, or specify one as `nil`, which will cause queries for that IP version to return `""`.
|
||||||
|
3. Please set an appropriate number of searchers based on your project's concurrency. This value is fixed during runtime; each query borrows a searcher from the pool to complete the operation and returns it afterward. If the pool is empty during borrowing, it will wait until a searcher becomes available.
|
||||||
|
4. If the cache policy is set to `service.BufferCache` (Full Memory Cache), a single-instance memory searcher is used by default. This implementation is natively concurrency-safe, and the specified number of searchers will be ignored.
|
||||||
|
5. If `Close` is called while the `Ip2Region` service is running, it will wait up to 10 seconds by default for searchers to be returned. You can also call `CloseTimeout` to define a custom maximum wait time.
|
||||||
|
|
||||||
|
### About the Query API
|
||||||
|
|
||||||
|
The location information query API prototypes are:
|
||||||
|
|
||||||
### 关于查询 API
|
|
||||||
定位信息查询 API 原型为:
|
|
||||||
```go
|
```go
|
||||||
SearchByStr(string) (string, error)
|
Search(string | []byte) (string, error)
|
||||||
Search([]byte) (string, error)
|
|
||||||
```
|
```
|
||||||
查询出错则 error 会包含具体的错误信息,查询成功会返回字符串的 `region` 信息,如果指定的 IP 查询不到则会返回空字符串 `""`。
|
|
||||||
|
|
||||||
### 关于 IPv4 / IPv6
|
If a query fails, the `error` will contain specific error details. If successful, it returns the `region` string. If the specified IP cannot be found, it returns an empty string `""`.
|
||||||
该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询,使用方式如下:
|
|
||||||
|
### About IPv4 / IPv6
|
||||||
|
|
||||||
|
This xdb query client implementation supports both IPv4 and IPv6 queries. Usage is as follows:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
// 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件,IP版本指定为 xdb.IPv4
|
// For IPv4: Set the xdb path to the v4 xdb file and specify the IP version as xdb.IPv4
|
||||||
dbPath := "../../data/ip2region_v4.xdb" // 或者你的 ipv4 xdb 的路径
|
dbPath := "../../data/ip2region_v4.xdb" // Or your ipv4 xdb path
|
||||||
version := xdb.IPv4
|
version := xdb.IPv4
|
||||||
|
|
||||||
// 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件,IP版本指定为 xdb.IPv6
|
// For IPv6: Set the xdb path to the v6 xdb file and specify the IP version as xdb.IPv6
|
||||||
dbPath = "../../data/ip2region_v6.xdb" // 或者你的 ipv6 xdb 路径
|
dbPath = "../../data/ip2region_v6.xdb" // Or your ipv6 xdb path
|
||||||
version = xdb.IPv6
|
version = xdb.IPv6
|
||||||
|
|
||||||
// dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致,不然查询执行的时候会报错
|
// The IP version of the xdb specified by dbPath must match the version specified, otherwise the query will throw an error.
|
||||||
// 备注:以下演示直接使用 dbPath 和 version 变量
|
// Note: The following demonstrations directly use the dbPath and version variables.
|
||||||
```
|
```
|
||||||
|
|
||||||
### 文件验证
|
### File Verification
|
||||||
建议您主动去验证 xdb 文件的适用性,因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件,验证可以避免运行过程中的一些不可预测的错误。
|
|
||||||
你不需要每次都去验证,例如在服务启动的时候,或者手动调用命令验证确认版本匹配即可,不要在每次创建的 Searcher 的时候运行验证,这样会影响查询的响应速度,尤其是高并发的使用场景。
|
It is recommended to actively verify the applicability of the xdb file. New features in later versions may make the current Searcher version incompatible with your xdb file. Verification helps avoid unpredictable errors during runtime.
|
||||||
|
You do not need to verify every time; for example, run it during service startup or manually via command line to confirm version matching. Do not run verification every time a Searcher is created, as this will impact query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
```go
|
```go
|
||||||
err := xdb.VerifyFromFile(dbPath)
|
err := xdb.VerifyFromFile(dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// err 包含的验证的错误
|
// err contains the verification error
|
||||||
return fmt.Errorf("xdb file verify: %w", err)
|
return fmt.Errorf("xdb file verify: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作
|
// The current Searcher can safely be used for query operations on the xdb specified by dbPath.
|
||||||
```
|
```
|
||||||
|
|
||||||
### 完全基于文件的查询
|
### Pure File-Based Query
|
||||||
|
|
||||||
```go
|
```go
|
||||||
import (
|
import (
|
||||||
|
|
@ -96,7 +107,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
// 通过 version 和 dbPath 创建完全基于文件的查询对象
|
// Create a pure file-based query object using version and dbPath
|
||||||
searcher, err := xdb.NewWithFileOnly(version, dbPath)
|
searcher, err := xdb.NewWithFileOnly(version, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to create searcher: %s\n", err.Error())
|
fmt.Printf("failed to create searcher: %s\n", err.Error())
|
||||||
|
|
@ -105,80 +116,82 @@ func main() {
|
||||||
|
|
||||||
defer searcher.Close()
|
defer searcher.Close()
|
||||||
|
|
||||||
// 定位信息查询:IPv4 或者 IPv6 的地址都支持
|
// Location info query: both IPv4 and IPv6 addresses are supported
|
||||||
var ip = "1.2.3.4" // IPv4
|
var ip = "1.2.3.4" // IPv4
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff" // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
var tStart = time.Now()
|
var tStart = time.Now()
|
||||||
region, err := searcher.SearchByStr(ip)
|
region, err := searcher.Search(ip)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to SearchIP(%s): %s\n", ip, err)
|
fmt.Printf("failed to Search(%s): %s\n", ip, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// IPv4 或者 IPv6 的定位信息
|
// IPv4 or IPv6 location information
|
||||||
fmt.Printf("{region: %s, took: %s}\n", region, time.Since(tStart))
|
fmt.Printf("{region: %s, took: %s}\n", region, time.Since(tStart))
|
||||||
|
|
||||||
// 备注:并发使用,每个 goroutine 需要创建一个独立的 searcher 对象。
|
// Note: For concurrent use, each goroutine needs to create an independent searcher object.
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存 `VectorIndex` 索引
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
You can pre-load the `vectorIndex` cache and store it as a global variable. Using the global `vectorIndex` whenever creating a searcher reduces a fixed IO operation, accelerating queries and reducing system IO pressure.
|
||||||
|
|
||||||
可以预先加载 `vectorIndex` 缓存,然后做成全局变量,每次创建 searcher 的时候使用全局的 `vectorIndex`,可以减少一次固定的 IO 操作从而加速查询,减少系统 io 压力。
|
|
||||||
```go
|
```go
|
||||||
// 1、从 dbPath 加载 VectorIndex 缓存,把下述 vIndex 变量全局到内存里面。
|
// 1. Load VectorIndex cache from dbPath and store the vIndex variable globally in memory.
|
||||||
vIndex, err := xdb.LoadVectorIndexFromFile(dbPath)
|
vIndex, err := xdb.LoadVectorIndexFromFile(dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to load vector index from `%s`: %s\n", dbPath, err)
|
fmt.Printf("failed to load vector index from `%s`: %s\n", dbPath, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2、用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
// 2. Create a query object with VectorIndex cache using the global vIndex.
|
||||||
searcher, err := xdb.NewWithVectorIndex(version, dbPath, vIndex)
|
searcher, err := xdb.NewWithVectorIndex(version, dbPath, vIndex)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to create searcher with vector index: %s\n", err)
|
fmt.Printf("failed to create searcher with vector index: %s\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// 备注:并发使用,全部 goroutine 共享全局的只读 vIndex 缓存,每个 goroutine 创建一个独立的 searcher 对象
|
// Note: For concurrent use, all goroutines share the global read-only vIndex cache, while each goroutine creates an independent searcher object.
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### Caching the entire `xdb` file
|
||||||
|
|
||||||
|
You can pre-load the entire xdb file into memory for full memory-based queries, similar to the previous memory search.
|
||||||
|
|
||||||
可以预先加载整个 ip2region.xdb 到内存,完全基于内存查询,类似于之前的 memory search 查询。
|
|
||||||
```go
|
```go
|
||||||
// 1、从 dbPath 加载整个 xdb 到内存
|
// 1. Load the entire xdb from dbPath into memory
|
||||||
cBuff, err := xdb.LoadContentFromFile(dbPath)
|
cBuff, err := xdb.LoadContentFromFile(dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to load content from `%s`: %s\n", dbPath, err)
|
fmt.Printf("failed to load content from `%s`: %s\n", dbPath, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2、用全局的 cBuff 创建完全基于内存的查询对象。
|
// 2. Create a fully memory-based query object using the global cBuff.
|
||||||
searcher, err := xdb.NewWithBuffer(version, cBuff)
|
searcher, err := xdb.NewWithBuffer(version, cBuff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("failed to create searcher with content: %s\n", err)
|
fmt.Printf("failed to create searcher with content: %s\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// 备注:并发使用,用整个 xdb 缓存创建的 searcher 对象可以安全用于并发。
|
// Note: For concurrent use, searcher objects created with the entire xdb cache can be safely used for concurrency.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
# Compile the test program
|
||||||
|
|
||||||
|
Compile to get the xdb_searcher executable through the following method:
|
||||||
|
|
||||||
# 编译测试程序
|
|
||||||
|
|
||||||
通过如下方式编译得到 xdb_searcher 可执行程序:
|
|
||||||
```bash
|
```bash
|
||||||
# 切换到 golang binding 根目录
|
# cd to the golang binding root directory first
|
||||||
make
|
make
|
||||||
```
|
```
|
||||||
|
|
||||||
|
# Query test
|
||||||
|
|
||||||
# 查询测试
|
### Query command
|
||||||
|
|
||||||
|
Test xdb queries using the `./xdb_searcher search` command:
|
||||||
|
|
||||||
### 查询命令
|
|
||||||
通过 `./xdb_searcher search` 命令来测试 xdb 的查询:
|
|
||||||
```bash
|
```bash
|
||||||
➜ golang git:(master) ✗ ./xdb_searcher search --help
|
➜ golang git:(master) ✗ ./xdb_searcher search --help
|
||||||
./xdb_searcher search [command options]
|
./xdb_searcher search [command options]
|
||||||
|
|
@ -190,14 +203,17 @@ options:
|
||||||
--help print this help menu
|
--help print this help menu
|
||||||
```
|
```
|
||||||
|
|
||||||
### 参数解析
|
### Parameter parsing
|
||||||
1. `v4-xdb`: IPv4 的 xdb 文件路径,默认为仓库中的 data/ip2region_v4.xdb
|
|
||||||
2. `v6-xdb`: IPv6 的 xdb 文件路径,默认为仓库中的 data/ip2region_v6.xdb
|
1. `v4-xdb`: IPv4 xdb file path, defaults to data/ip2region_v4.xdb in the repository
|
||||||
3. `v4-cache-policy`: v4 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
2. `v6-xdb`: IPv6 xdb file path, defaults to data/ip2region_v6.xdb in the repository
|
||||||
4. `v6-cache-policy`: v6 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
3. `v4-cache-policy`: Cache policy used for v4 queries, defaults to `vectorIndex`, options: file/vectorIndex/content
|
||||||
|
4. `v6-cache-policy`: Cache policy used for v6 queries, defaults to `vectorIndex`, options: file/vectorIndex/content
|
||||||
|
|
||||||
|
### Test Demo
|
||||||
|
|
||||||
|
Example: Perform query tests using the default data/ip2region_v4.xdb and data/ip2region_v6.xdb:
|
||||||
|
|
||||||
### 测试 Demo
|
|
||||||
例如:使用默认的 data/ip2region_v4.xdb 和 data/ip2region_v6.xdb 进行查询测试:
|
|
||||||
```bash
|
```bash
|
||||||
➜ golang git:(master) ✗ ./xdb_searcher search
|
➜ golang git:(master) ✗ ./xdb_searcher search
|
||||||
ip2region search service test program
|
ip2region search service test program
|
||||||
|
|
@ -205,18 +221,21 @@ ip2region search service test program
|
||||||
+-v6 db: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex)
|
+-v6 db: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
ip2region>> 1.2.3.4
|
ip2region>> 1.2.3.4
|
||||||
{region: 美国|华盛顿|0|谷歌, took: 69.088µs}
|
{region: Australia|Queensland|Brisbane|0|AU, took: 50.216µs}
|
||||||
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, took: 67.756µs}
|
{region: 中国|广东省|深圳市|电信|CN, took: 100.606µs}
|
||||||
ip2region>>
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, took: 99.078µs}
|
||||||
```
|
```
|
||||||
输入 v4 或者 v6 的 IP 地址即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
|
|
||||||
|
|
||||||
|
Enter an IPv4 or IPv6 address to perform query tests. You can also set `cache-policy` to file/vectorIndex/content respectively to test the performance of the three different cache implementations.
|
||||||
|
|
||||||
# bench 测试
|
# bench test
|
||||||
|
|
||||||
|
### Test command
|
||||||
|
|
||||||
|
Perform automatic bench testing using the `xdb_searcher bench` command. This ensures that both the program and the `xdb` file are free of errors, and provides average query performance through a large number of queries:
|
||||||
|
|
||||||
### 测试命令
|
|
||||||
通过 `xdb_searcher bench` 命令来进行自动 bench 测试,一方面确保程序和 `xdb` 文件都没有错误,另一方面通过大量的查询得到平均查询性能:
|
|
||||||
```bash
|
```bash
|
||||||
➜ golang git:(fr_xdb_ipv6) ./xdb_searcher bench
|
➜ golang git:(fr_xdb_ipv6) ./xdb_searcher bench
|
||||||
./xdb_searcher bench [command options]
|
./xdb_searcher bench [command options]
|
||||||
|
|
@ -227,19 +246,23 @@ options:
|
||||||
```
|
```
|
||||||
|
|
||||||
### v4 bench
|
### v4 bench
|
||||||
例如:通过 data/ip2region_v4.xdb 和 data/ipv4_source.txt 进行 ipv4 的 bench 测试:
|
|
||||||
|
Example: Perform ipv4 bench testing using data/ip2region_v4.xdb and data/ipv4_source.txt:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
### v6 bench
|
### v6 bench
|
||||||
例如:通过 data/ip2region_v6.xdb 和 data/ipv6_source.txt 进行 ipv6 的 bench 测试:
|
|
||||||
|
Example: Perform ipv6 bench testing using data/ip2region_v6.xdb and data/ipv6_source.txt:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
可以设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 不同缓存实现机制的效率。
|
You can set the `cache-policy` parameter to test the efficiency of file/vectorIndex/content cache mechanisms respectively.
|
||||||
|
|
||||||
*请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件*。
|
*Please note that the src file used for bench needs to be the same source file used to generate the corresponding xdb file*.
|
||||||
|
|
||||||
bench 程序会逐行读取 `src` 指定的源IP文件,然后每个 IP 段选取 5 个固定位置的 IP 进行测试,以确保查询的 region 信息和原始的 region 信息是相同。测试途中没有调试信息的输出,有错误会打印错误信息并且终止运行,所以看到 `Bench finished` 就表示 bench 成功了,cost 是表示每次查询操作的平均时间(ns)。
|
The bench program will read the source IP file specified by `src` line by line, then select the start and end IPs from each IP segment for testing to ensure that the queried region information matches the original region information. There is no debug information output during the test; if an error occurs, the error message will be printed and execution will terminate. Seeing `Bench finished` indicates the bench was successful. Cost represents the average time for each query operation (ns).
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,247 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region golang 查询客户端
|
||||||
|
|
||||||
|
# 使用方式
|
||||||
|
|
||||||
|
### package 获取
|
||||||
|
```bash
|
||||||
|
go get github.com/lionsoul2014/ip2region/binding/golang
|
||||||
|
```
|
||||||
|
|
||||||
|
### 关于查询服务
|
||||||
|
从 `3.11.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务,**建议优先使用该方式来进行查询调用**,具体使用方式如下:
|
||||||
|
```go
|
||||||
|
import "github.com/lionsoul2014/ip2region/binding/golang/service"
|
||||||
|
|
||||||
|
// 1, 创建 v4 的配置:指定缓存策略和 v4 的 xdb 文件路径
|
||||||
|
// 参数1: 缓存策略, options: service.NoCache / service.VIndexCache / service.BufferCache
|
||||||
|
// 参数2: xdb 文件路径
|
||||||
|
// 参数3: 初始化的查询器数量
|
||||||
|
v4Config, err := service.NewV4Config(service.VIndexCache, "ip2region v4 xdb path", 20)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create v4 config: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2, 创建 v6 的配置:指定缓存策略和 v6 的 xdb 文件路径
|
||||||
|
v6Config, err := service.NewV6Config(service.VIndexCache, "ip2region v6 xdb path", 20)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create v6 config: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3,通过上述配置创建 Ip2Region 查询服务
|
||||||
|
ip2region, err := service.NewIp2Region(v4Config, v6Config)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create ip2region service: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4,导出 ip2region 服务进行双版本的IP地址的并发查询,例如:
|
||||||
|
v4Region, err := ip2region.Search("113.92.157.29") // 进行 IPv4 查询
|
||||||
|
v6Region, err := ip2region.Search("240e:3b7:3272:d8d0:db09:c067:8d59:539e") // 进行 IPv6 查询
|
||||||
|
|
||||||
|
|
||||||
|
// 5,在服务需要关闭的时候,同时关闭 ip2region 查询服务
|
||||||
|
ip2region.Close()
|
||||||
|
```
|
||||||
|
|
||||||
|
##### `Ip2Region` 查询备注:
|
||||||
|
1. 该查询服务的 API 并发安全且同时支持 IPv4 和 Ipv6 的地址,内部实现会自动判断。
|
||||||
|
2. v4 和 v6 的配置需要单独创建,可以给 v4 和 v6 设置使用不同的缓存策略,也可以指定其中一个为 `nil` 则该版本的 IP 地址查询都会返回 `""`。
|
||||||
|
3. 请结合您的项目的并发数设置一个合适的查询器数量,这个值在运行过程中是固定的,每次查询会从池子里租借一个查询器来完成查询操作,查询完成后再归还回去,如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务。
|
||||||
|
4. 如果配置设置的缓存策略为 `service.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器,该实现天生并发安全,此时指定的查询器数量无效。
|
||||||
|
5. 如果 `Ip2Region` 查询器在提供服务期间,调用 Close 默认会最大等待 10 秒钟来等待尽量多的查询器归还,也可以调用 `CloseTimeout` 来自定义最长等待时间。
|
||||||
|
|
||||||
|
|
||||||
|
### 关于查询 API
|
||||||
|
定位信息查询 API 原型为:
|
||||||
|
```go
|
||||||
|
Search(string | []byte) (string, error)
|
||||||
|
```
|
||||||
|
查询出错则 error 会包含具体的错误信息,查询成功会返回字符串的 `region` 信息,如果指定的 IP 查询不到则会返回空字符串 `""`。
|
||||||
|
|
||||||
|
### 关于 IPv4 / IPv6
|
||||||
|
该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询,使用方式如下:
|
||||||
|
```go
|
||||||
|
// 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件,IP版本指定为 xdb.IPv4
|
||||||
|
dbPath := "../../data/ip2region_v4.xdb" // 或者你的 ipv4 xdb 的路径
|
||||||
|
version := xdb.IPv4
|
||||||
|
|
||||||
|
// 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件,IP版本指定为 xdb.IPv6
|
||||||
|
dbPath = "../../data/ip2region_v6.xdb" // 或者你的 ipv6 xdb 路径
|
||||||
|
version = xdb.IPv6
|
||||||
|
|
||||||
|
// dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致,不然查询执行的时候会报错
|
||||||
|
// 备注:以下演示直接使用 dbPath 和 version 变量
|
||||||
|
```
|
||||||
|
|
||||||
|
### 文件验证
|
||||||
|
建议您主动去验证 xdb 文件的适用性,因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件,验证可以避免运行过程中的一些不可预测的错误。
|
||||||
|
你不需要每次都去验证,例如在服务启动的时候,或者手动调用命令验证确认版本匹配即可,不要在每次创建的 Searcher 的时候运行验证,这样会影响查询的响应速度,尤其是高并发的使用场景。
|
||||||
|
```go
|
||||||
|
err := xdb.VerifyFromFile(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
// err 包含的验证的错误
|
||||||
|
return fmt.Errorf("xdb file verify: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作
|
||||||
|
```
|
||||||
|
|
||||||
|
### 完全基于文件的查询
|
||||||
|
|
||||||
|
```go
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/lionsoul2014/ip2region/binding/golang/xdb"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// 通过 version 和 dbPath 创建完全基于文件的查询对象
|
||||||
|
searcher, err := xdb.NewWithFileOnly(version, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to create searcher: %s\n", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer searcher.Close()
|
||||||
|
|
||||||
|
// 定位信息查询:IPv4 或者 IPv6 的地址都支持
|
||||||
|
var ip = "1.2.3.4" // IPv4
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
|
var tStart = time.Now()
|
||||||
|
region, err := searcher.Search(ip)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to Search(%s): %s\n", ip, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// IPv4 或者 IPv6 的定位信息
|
||||||
|
fmt.Printf("{region: %s, took: %s}\n", region, time.Since(tStart))
|
||||||
|
|
||||||
|
// 备注:并发使用,每个 goroutine 需要创建一个独立的 searcher 对象。
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 缓存 `VectorIndex` 索引
|
||||||
|
|
||||||
|
可以预先加载 `vectorIndex` 缓存,然后做成全局变量,每次创建 searcher 的时候使用全局的 `vectorIndex`,可以减少一次固定的 IO 操作从而加速查询,减少系统 io 压力。
|
||||||
|
```go
|
||||||
|
// 1、从 dbPath 加载 VectorIndex 缓存,把下述 vIndex 变量全局到内存里面。
|
||||||
|
vIndex, err := xdb.LoadVectorIndexFromFile(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to load vector index from `%s`: %s\n", dbPath, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2、用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
||||||
|
searcher, err := xdb.NewWithVectorIndex(version, dbPath, vIndex)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to create searcher with vector index: %s\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 备注:并发使用,全部 goroutine 共享全局的只读 vIndex 缓存,每个 goroutine 创建一个独立的 searcher 对象
|
||||||
|
```
|
||||||
|
|
||||||
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
|
可以预先加载整个 xdb 文件到内存,完全基于内存查询,类似于之前的 memory search 查询。
|
||||||
|
```go
|
||||||
|
// 1、从 dbPath 加载整个 xdb 到内存
|
||||||
|
cBuff, err := xdb.LoadContentFromFile(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to load content from `%s`: %s\n", dbPath, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2、用全局的 cBuff 创建完全基于内存的查询对象。
|
||||||
|
searcher, err := xdb.NewWithBuffer(version, cBuff)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to create searcher with content: %s\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// 备注:并发使用,用整个 xdb 缓存创建的 searcher 对象可以安全用于并发。
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# 编译测试程序
|
||||||
|
|
||||||
|
通过如下方式编译得到 xdb_searcher 可执行程序:
|
||||||
|
```bash
|
||||||
|
# 切换到 golang binding 根目录
|
||||||
|
make
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
# 查询测试
|
||||||
|
|
||||||
|
### 查询命令
|
||||||
|
通过 `./xdb_searcher search` 命令来测试 xdb 的查询:
|
||||||
|
```bash
|
||||||
|
➜ golang git:(master) ✗ ./xdb_searcher search --help
|
||||||
|
./xdb_searcher search [command options]
|
||||||
|
options:
|
||||||
|
--v4-db string ip2region v4 binary xdb file path
|
||||||
|
--v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content
|
||||||
|
--v6-db string ip2region v6 binary xdb file path
|
||||||
|
--v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content
|
||||||
|
--help print this help menu
|
||||||
|
```
|
||||||
|
|
||||||
|
### 参数解析
|
||||||
|
1. `v4-xdb`: IPv4 的 xdb 文件路径,默认为仓库中的 data/ip2region_v4.xdb
|
||||||
|
2. `v6-xdb`: IPv6 的 xdb 文件路径,默认为仓库中的 data/ip2region_v6.xdb
|
||||||
|
3. `v4-cache-policy`: v4 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
||||||
|
4. `v6-cache-policy`: v6 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
||||||
|
|
||||||
|
### 测试 Demo
|
||||||
|
例如:使用默认的 data/ip2region_v4.xdb 和 data/ip2region_v6.xdb 进行查询测试:
|
||||||
|
```bash
|
||||||
|
➜ golang git:(master) ✗ ./xdb_searcher search
|
||||||
|
ip2region search service test program
|
||||||
|
+-v4 db: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex)
|
||||||
|
+-v6 db: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, took: 50.216µs}
|
||||||
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, took: 100.606µs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, took: 99.078µs}
|
||||||
|
```
|
||||||
|
输入 v4 或者 v6 的 IP 地址即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
|
||||||
|
|
||||||
|
|
||||||
|
# bench 测试
|
||||||
|
|
||||||
|
### 测试命令
|
||||||
|
通过 `xdb_searcher bench` 命令来进行自动 bench 测试,一方面确保程序和 `xdb` 文件都没有错误,另一方面通过大量的查询得到平均查询性能:
|
||||||
|
```bash
|
||||||
|
➜ golang git:(fr_xdb_ipv6) ./xdb_searcher bench
|
||||||
|
./xdb_searcher bench [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--src string source ip text file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
### v4 bench
|
||||||
|
例如:通过 data/ip2region_v4.xdb 和 data/ipv4_source.txt 进行 ipv4 的 bench 测试:
|
||||||
|
```bash
|
||||||
|
./xdb_searcher bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### v6 bench
|
||||||
|
例如:通过 data/ip2region_v6.xdb 和 data/ipv6_source.txt 进行 ipv6 的 bench 测试:
|
||||||
|
```bash
|
||||||
|
./xdb_searcher bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
可以设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 不同缓存实现机制的效率。
|
||||||
|
|
||||||
|
*请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件*。
|
||||||
|
|
||||||
|
bench 程序会逐行读取 `src` 指定的源IP文件,然后每个 IP 段选取开始和结束的 IP 进行测试,以确保查询的 region 信息和原始的 region 信息是相同。测试途中没有调试信息的输出,有错误会打印错误信息并且终止运行,所以看到 `Bench finished` 就表示 bench 成功了,cost 是表示每次查询操作的平均时间(ns)。
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
module github.com/lionsoul2014/ip2region/binding/golang
|
module github.com/lionsoul2014/ip2region/binding/golang
|
||||||
|
|
||||||
go 1.17
|
go 1.18
|
||||||
|
|
||||||
require github.com/mitchellh/go-homedir v1.1.0
|
require github.com/mitchellh/go-homedir v1.1.0
|
||||||
|
|
@ -239,7 +239,7 @@ type 'quit' to exit
|
||||||
}
|
}
|
||||||
|
|
||||||
tStart := time.Now()
|
tStart := time.Now()
|
||||||
region, err := ip2region.SearchByStr(line)
|
region, err := ip2region.Search(line)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("\x1b[0;31m{err: %s}\x1b[0m\n", err.Error())
|
fmt.Printf("\x1b[0;31m{err: %s}\x1b[0m\n", err.Error())
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,7 @@ func newConfig(cachePolicy int, ipVersion *xdb.Version, xdbPath string, searcher
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
defer handle.Close()
|
||||||
|
|
||||||
// 1, verify the xdb
|
// 1, verify the xdb
|
||||||
err = xdb.Verify(handle)
|
err = xdb.Verify(handle)
|
||||||
|
|
|
||||||
|
|
@ -118,16 +118,21 @@ func NewIp2RegionWithPath(v4XdbPath string, v6XdbPath string) (*Ip2Region, error
|
||||||
return NewIp2Region(v4Config, v6Config)
|
return NewIp2Region(v4Config, v6Config)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ip2r *Ip2Region) SearchByStr(ipStr string) (string, error) {
|
func (ip2r *Ip2Region) Search(ip any) (string, error) {
|
||||||
ipBytes, err := xdb.ParseIP(ipStr)
|
var err error
|
||||||
if err != nil {
|
var ipBytes []byte
|
||||||
return "", err
|
switch v := ip.(type) {
|
||||||
|
case string:
|
||||||
|
ipBytes, err = xdb.ParseIP(v)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("parse ip %s: %w", v, err)
|
||||||
|
}
|
||||||
|
case []byte:
|
||||||
|
ipBytes = v
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("invalid ip value type %s", v)
|
||||||
}
|
}
|
||||||
|
|
||||||
return ip2r.Search(ipBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (ip2r *Ip2Region) Search(ipBytes []byte) (string, error) {
|
|
||||||
if l := len(ipBytes); l == 4 {
|
if l := len(ipBytes); l == 4 {
|
||||||
return ip2r.v4Search(ipBytes)
|
return ip2r.v4Search(ipBytes)
|
||||||
} else if l == 16 {
|
} else if l == 16 {
|
||||||
|
|
|
||||||
|
|
@ -245,12 +245,12 @@ func TestConcurrentCall(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if l := len(ipBytes); l == 4 {
|
if l := len(ipBytes); l == 4 {
|
||||||
if region != "中国|广东省|深圳市|电信" {
|
if region != "中国|广东省|深圳市|电信|CN" {
|
||||||
fmt.Print("Error: region not equals")
|
fmt.Print("Error: region not equals")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if region != "中国|广东省|深圳市|家庭宽带" {
|
if region != "中国|广东省|深圳市|电信|CN" {
|
||||||
fmt.Print("Error: region not equals")
|
fmt.Print("Error: region not equals")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ func TestV4SearcherPool(t *testing.T) {
|
||||||
ipString := "219.133.110.197"
|
ipString := "219.133.110.197"
|
||||||
for i := 0; i < 20; i++ {
|
for i := 0; i < 20; i++ {
|
||||||
searcher := searcherPool.BorrowSearcher()
|
searcher := searcherPool.BorrowSearcher()
|
||||||
region, err := searcher.SearchByStr(ipString)
|
region, err := searcher.Search(ipString)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to search(%s): %s", ipString, err)
|
t.Fatalf("failed to search(%s): %s", ipString, err)
|
||||||
}
|
}
|
||||||
|
|
@ -53,7 +53,7 @@ func TestV6SearcherPool(t *testing.T) {
|
||||||
ipString := "240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6"
|
ipString := "240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6"
|
||||||
for i := 0; i < 20; i++ {
|
for i := 0; i < 20; i++ {
|
||||||
searcher := searcherPool.BorrowSearcher()
|
searcher := searcherPool.BorrowSearcher()
|
||||||
region, err := searcher.SearchByStr(ipString)
|
region, err := searcher.Search(ipString)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to search(%s): %s", ipString, err)
|
t.Fatalf("failed to search(%s): %s", ipString, err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ type Header struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHeader(input []byte) (*Header, error) {
|
func NewHeader(input []byte) (*Header, error) {
|
||||||
if len(input) < 16 {
|
if len(input) < 20 {
|
||||||
return nil, fmt.Errorf("invalid input buffer")
|
return nil, fmt.Errorf("invalid input buffer")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,12 +14,13 @@ package xdb
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Searcher struct {
|
type Searcher struct {
|
||||||
version *Version
|
version *Version
|
||||||
handle *os.File
|
dbReader io.ReadSeekCloser
|
||||||
|
|
||||||
ioCount int
|
ioCount int
|
||||||
|
|
||||||
|
|
@ -65,14 +66,31 @@ func NewSearcher(version *Version, dbFile string, vIndex []byte, cBuff []byte) (
|
||||||
|
|
||||||
return &Searcher{
|
return &Searcher{
|
||||||
version: version,
|
version: version,
|
||||||
handle: handle,
|
dbReader: handle,
|
||||||
vectorIndex: vIndex,
|
vectorIndex: vIndex,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func INewSearcher(version *Version, dbReader io.ReadSeekCloser, vIndex []byte, cBuff []byte) *Searcher {
|
||||||
|
// content buff first
|
||||||
|
if cBuff != nil {
|
||||||
|
return &Searcher{
|
||||||
|
version: version,
|
||||||
|
vectorIndex: nil,
|
||||||
|
contentBuff: cBuff,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return &Searcher{
|
||||||
|
version: version,
|
||||||
|
dbReader: dbReader,
|
||||||
|
vectorIndex: vIndex,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Searcher) Close() {
|
func (s *Searcher) Close() {
|
||||||
if s.handle != nil {
|
if s.dbReader != nil {
|
||||||
err := s.handle.Close()
|
err := s.dbReader.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// do error log here ?
|
// do error log here ?
|
||||||
}
|
}
|
||||||
|
|
@ -89,20 +107,24 @@ func (s *Searcher) GetIOCount() int {
|
||||||
return s.ioCount
|
return s.ioCount
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchByStr find the region for the specified ip string
|
// Search the region for the specified string or bytes ip address
|
||||||
func (s *Searcher) SearchByStr(str string) (string, error) {
|
func (s *Searcher) Search(ip any) (string, error) {
|
||||||
ip, err := ParseIP(str)
|
var err error
|
||||||
if err != nil {
|
var ipBytes []byte
|
||||||
return "", err
|
switch v := ip.(type) {
|
||||||
|
case string:
|
||||||
|
ipBytes, err = ParseIP(v)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("parse ip %s: %w", v, err)
|
||||||
|
}
|
||||||
|
case []byte:
|
||||||
|
ipBytes = v
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("invalid ip value type %s", v)
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.Search(ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search find the region for the specified long ip
|
|
||||||
func (s *Searcher) Search(ip []byte) (string, error) {
|
|
||||||
// ip version check
|
// ip version check
|
||||||
if len(ip) != s.version.Bytes {
|
if len(ipBytes) != s.version.Bytes {
|
||||||
return "", fmt.Errorf("invalid ip address(%s expected)", s.version.Name)
|
return "", fmt.Errorf("invalid ip address(%s expected)", s.version.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -110,7 +132,7 @@ func (s *Searcher) Search(ip []byte) (string, error) {
|
||||||
s.ioCount = 0
|
s.ioCount = 0
|
||||||
|
|
||||||
// locate the segment index block based on the vector index
|
// locate the segment index block based on the vector index
|
||||||
var il0, il1 = int(ip[0]), int(ip[1])
|
var il0, il1 = int(ipBytes[0]), int(ipBytes[1])
|
||||||
var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize
|
var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize
|
||||||
var sPtr, ePtr = uint32(0), uint32(0)
|
var sPtr, ePtr = uint32(0), uint32(0)
|
||||||
if s.vectorIndex != nil {
|
if s.vectorIndex != nil {
|
||||||
|
|
@ -132,9 +154,14 @@ func (s *Searcher) Search(ip []byte) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// fmt.Printf("sPtr=%d, ePtr=%d\n", sPtr, ePtr)
|
// fmt.Printf("sPtr=%d, ePtr=%d\n", sPtr, ePtr)
|
||||||
|
// @Note: ptr validate, zero ptr means source data missing
|
||||||
|
// so we could just stop here and return an empty string.
|
||||||
|
if sPtr == 0 || ePtr == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
// binary search the segment index to get the region
|
// binary search the segment index to get the region
|
||||||
var bytes, dBytes = len(ip), len(ip) << 1
|
var bytes, dBytes = len(ipBytes), len(ipBytes) << 1
|
||||||
var segIndexSize = uint32(s.version.SegmentIndexSize)
|
var segIndexSize = uint32(s.version.SegmentIndexSize)
|
||||||
var dataLen, dataPtr = 0, uint32(0)
|
var dataLen, dataPtr = 0, uint32(0)
|
||||||
var buff = make([]byte, segIndexSize)
|
var buff = make([]byte, segIndexSize)
|
||||||
|
|
@ -148,9 +175,9 @@ func (s *Searcher) Search(ip []byte) (string, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// decode the data step by step to reduce the unnecessary operations
|
// decode the data step by step to reduce the unnecessary operations
|
||||||
if s.version.IPCompare(ip, buff[0:bytes]) < 0 {
|
if s.version.IPCompare(ipBytes, buff[0:bytes]) < 0 {
|
||||||
h = m - 1
|
h = m - 1
|
||||||
} else if s.version.IPCompare(ip, buff[bytes:dBytes]) > 0 {
|
} else if s.version.IPCompare(ipBytes, buff[bytes:dBytes]) > 0 {
|
||||||
l = m + 1
|
l = m + 1
|
||||||
} else {
|
} else {
|
||||||
dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:]))
|
dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:]))
|
||||||
|
|
@ -166,7 +193,7 @@ func (s *Searcher) Search(ip []byte) (string, error) {
|
||||||
|
|
||||||
// load and return the region data
|
// load and return the region data
|
||||||
var regionBuff = make([]byte, dataLen)
|
var regionBuff = make([]byte, dataLen)
|
||||||
err := s.read(int64(dataPtr), regionBuff)
|
err = s.read(int64(dataPtr), regionBuff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("read region at %d: %w", dataPtr, err)
|
return "", fmt.Errorf("read region at %d: %w", dataPtr, err)
|
||||||
}
|
}
|
||||||
|
|
@ -184,13 +211,13 @@ func (s *Searcher) read(offset int64, buff []byte) error {
|
||||||
return fmt.Errorf("incomplete read: readed bytes should be %d", len(buff))
|
return fmt.Errorf("incomplete read: readed bytes should be %d", len(buff))
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
_, err := s.handle.Seek(offset, 0)
|
_, err := s.dbReader.Seek(offset, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("seek to %d: %w", offset, err)
|
return fmt.Errorf("seek to %d: %w", offset, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.ioCount++
|
s.ioCount++
|
||||||
rLen, err := s.handle.Read(buff)
|
rLen, err := s.dbReader.Read(buff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("handle read: %w", err)
|
return fmt.Errorf("handle read: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -82,6 +82,59 @@ func IPSubOne(ip []byte) []byte {
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IPSub Sub the spcecified two byte ip
|
||||||
|
func IPSub(sip, eip []byte) ([]byte, error) {
|
||||||
|
if len(sip) != len(eip) {
|
||||||
|
return []byte{}, fmt.Errorf("length of the two ips are not the same")
|
||||||
|
}
|
||||||
|
|
||||||
|
var carry uint16 = 0
|
||||||
|
var result = make([]byte, len(sip)+1)
|
||||||
|
|
||||||
|
for i := len(sip) - 1; i >= 0; i-- {
|
||||||
|
sum := uint16(sip[i]) + uint16(eip[i]) + carry
|
||||||
|
result[i+1] = byte(sum) // Store standard 8-bit result
|
||||||
|
carry = sum >> 8 // Extract the 1-bit carry for the next byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// check and append the carry
|
||||||
|
if carry > 0 {
|
||||||
|
result[0] = byte(carry)
|
||||||
|
return result, nil
|
||||||
|
} else {
|
||||||
|
return result[1:], nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// IPHalf get the half value of an input byte ip
|
||||||
|
func IPHalf(ip []byte) []byte {
|
||||||
|
var length = len(ip)
|
||||||
|
var result = make([]byte, length)
|
||||||
|
// Tracks the bit falling off from the previous byte
|
||||||
|
var carry byte = 0
|
||||||
|
|
||||||
|
for i := 0; i < length; i++ {
|
||||||
|
// 1. Shift current byte right by 1
|
||||||
|
// 2. Or (|) with the carry from the previous byte (shifted to the MSB position)
|
||||||
|
result[i] = (ip[i] >> 1) | (carry << 7)
|
||||||
|
|
||||||
|
// 3. Capture the Least Significant Bit (LSB) to use as carry for the next byte
|
||||||
|
carry = ip[i] & 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// IPMiddle get the middle value of two input ip address
|
||||||
|
func IPMiddle(sip, eip []byte) ([]byte, error) {
|
||||||
|
buf, err := IPSub(sip, eip)
|
||||||
|
if err != nil {
|
||||||
|
return []byte{}, fmt.Errorf("IPSub(%s, %s): %w", IP2String(sip), IP2String(eip), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return IPHalf(buf), nil
|
||||||
|
}
|
||||||
|
|
||||||
// Verify if the current Searcher could be used to search the specified xdb file.
|
// Verify if the current Searcher could be used to search the specified xdb file.
|
||||||
// Why do we need this check ?
|
// Why do we need this check ?
|
||||||
// The future features of the xdb impl may cause the current searcher not able to work properly.
|
// The future features of the xdb impl may cause the current searcher not able to work properly.
|
||||||
|
|
@ -132,7 +185,7 @@ func VerifyFromFile(dbFile string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadHeader load the header info from the specified handle
|
// LoadHeader load the header info from the specified handle
|
||||||
func LoadHeader(handle *os.File) (*Header, error) {
|
func LoadHeader(handle io.ReadSeeker) (*Header, error) {
|
||||||
_, err := handle.Seek(0, 0)
|
_, err := handle.Seek(0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("seek to the header: %w", err)
|
return nil, fmt.Errorf("seek to the header: %w", err)
|
||||||
|
|
@ -173,7 +226,7 @@ func LoadHeaderFromBuff(cBuff []byte) (*Header, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadVectorIndex util function to load the vector index from the specified file handle
|
// LoadVectorIndex util function to load the vector index from the specified file handle
|
||||||
func LoadVectorIndex(handle *os.File) ([]byte, error) {
|
func LoadVectorIndex(handle io.ReadSeeker) ([]byte, error) {
|
||||||
// load all the vector index block
|
// load all the vector index block
|
||||||
_, err := handle.Seek(HeaderInfoLength, 0)
|
_, err := handle.Seek(HeaderInfoLength, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -210,32 +263,14 @@ func LoadVectorIndexFromFile(dbFile string) ([]byte, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadContent load the whole xdb content from the specified file handle
|
// LoadContent load the whole xdb content from the specified file handle
|
||||||
func LoadContent(handle *os.File) ([]byte, error) {
|
func LoadContent(handle io.ReadSeeker) ([]byte, error) {
|
||||||
// get file size
|
|
||||||
fi, err := handle.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("stat: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
size := fi.Size()
|
|
||||||
|
|
||||||
// seek to the head of the file
|
// seek to the head of the file
|
||||||
_, err = handle.Seek(0, 0)
|
_, err := handle.Seek(0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("seek to get xdb file length: %w", err)
|
return nil, fmt.Errorf("seek to get xdb file length: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var buff = make([]byte, size)
|
return io.ReadAll(handle)
|
||||||
rLen, err := handle.Read(buff)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if rLen != len(buff) {
|
|
||||||
return nil, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff))
|
|
||||||
}
|
|
||||||
|
|
||||||
return buff, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadContentFromFile load the whole xdb content from the specified db file path
|
// LoadContentFromFile load the whole xdb content from the specified db file path
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@
|
||||||
package xdb
|
package xdb
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/binary"
|
||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
@ -41,6 +42,102 @@ func TestIPCompare(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIPSub(t *testing.T) {
|
||||||
|
var strToSub = "1.2.3.4"
|
||||||
|
bytesToSub, err := ParseIP(strToSub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to parse ip %s", strToSub)
|
||||||
|
}
|
||||||
|
var intToSub = int(binary.BigEndian.Uint32(bytesToSub))
|
||||||
|
t.Logf("to sub ip: %d -> %s", intToSub, strToSub)
|
||||||
|
|
||||||
|
counter := 0
|
||||||
|
buf := make([]byte, 4)
|
||||||
|
for i := 0; i < 0x2FFFFFFF; i++ {
|
||||||
|
binary.BigEndian.PutUint32(buf, uint32(i))
|
||||||
|
subVal, err := IPSub(buf, bytesToSub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to IPSub(%s,%s): %s", IP2String(buf), strToSub, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// do it as two integers
|
||||||
|
byteSub := int(binary.BigEndian.Uint32(subVal))
|
||||||
|
intSub := i + intToSub
|
||||||
|
if byteSub != intSub {
|
||||||
|
t.Fatal("byte and int sub value are not the same")
|
||||||
|
}
|
||||||
|
|
||||||
|
counter++
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("test done with %d ips", counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIPHalf(t *testing.T) {
|
||||||
|
var buf = make([]byte, 4)
|
||||||
|
for i := 0; i < 0xFFFFFFFF; i++ {
|
||||||
|
binary.BigEndian.PutUint32(buf, uint32(i))
|
||||||
|
half := IPHalf(buf)
|
||||||
|
|
||||||
|
// do it as two integers
|
||||||
|
byteMiddle := binary.BigEndian.Uint32(half)
|
||||||
|
intMidle := i >> 1
|
||||||
|
if byteMiddle != uint32(intMidle) {
|
||||||
|
t.Fatal("byte middle and int middle are not the same")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubOverflow(t *testing.T) {
|
||||||
|
var ip1Str = "255.255.255.250"
|
||||||
|
ip1Bytes, err := ParseIP(ip1Str)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to ParseIP(%s): %s", ip1Str, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buff = make([]byte, 4)
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
binary.BigEndian.PutUint32(buff, uint32(i))
|
||||||
|
ipSub, err := IPSub(ip1Bytes, buff)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to IPSub(%s, %s): %s", ip1Str, IP2String(buff), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("IPSub(%s, %s) = %+v", ip1Str, IP2String(buff), ipSub)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIPMiddle(t *testing.T) {
|
||||||
|
var sIPStr = "0.0.0.0"
|
||||||
|
sBytes, err := ParseIP(sIPStr)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to parse ip %s", sIPStr)
|
||||||
|
}
|
||||||
|
var sInt = int(binary.BigEndian.Uint32(sBytes))
|
||||||
|
t.Logf("start ip: %d -> %s", sInt, sIPStr)
|
||||||
|
|
||||||
|
counter := 0
|
||||||
|
buf := make([]byte, 4)
|
||||||
|
for i := 0; i < 0x0FFFFFFF; i++ {
|
||||||
|
binary.BigEndian.PutUint32(buf, uint32(i))
|
||||||
|
midVal, err := IPMiddle(sBytes, buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to IPMiddle(%s,%s): %s", sIPStr, IP2String(buf), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// do it as two integers
|
||||||
|
byteMid := int(binary.BigEndian.Uint32(midVal))
|
||||||
|
intMid := (sInt + i) >> 1
|
||||||
|
if byteMid != intMid {
|
||||||
|
t.Fatal("byte and int middle value are not the same")
|
||||||
|
}
|
||||||
|
|
||||||
|
counter++
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("test done with %d ips", counter)
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadVectorIndex(t *testing.T) {
|
func TestLoadVectorIndex(t *testing.T) {
|
||||||
vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region_v4.xdb")
|
vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region_v4.xdb")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -1,105 +1,120 @@
|
||||||
# ip2region xdb java 查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
# 使用方式
|
# ip2region java Query Client
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### Maven Repository:
|
||||||
|
|
||||||
### maven 仓库:
|
|
||||||
```xml
|
```xml
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.lionsoul</groupId>
|
<groupId>org.lionsoul</groupId>
|
||||||
<artifactId>ip2region</artifactId>
|
<artifactId>ip2region</artifactId>
|
||||||
<version>3.3.4</version>
|
<version>3.3.7</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
### 关于查询服务
|
### About Query Service
|
||||||
从 `3.2.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务,**建议优先使用该方式来进行查询调用**,具体使用方式如下:
|
|
||||||
|
Starting from version `3.2.0`, a dual-protocol compatible and concurrency-safe `Ip2Region` query service is provided. **It is recommended to prioritize this method for query calls.** The specific usage is as follows:
|
||||||
|
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.service.Config;
|
import org.lionsoul.ip2region.service.Config;
|
||||||
import org.lionsoul.ip2region.service.Ip2Region;
|
import org.lionsoul.ip2region.service.Ip2Region;
|
||||||
|
|
||||||
// 1, 创建 v4 的配置:指定缓存策略和 v4 的 xdb 文件路径
|
// 1. Create v4 configuration: specify cache policy and v4 xdb file path
|
||||||
final Config v4Config = Config.custom()
|
final Config v4Config = Config.custom()
|
||||||
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
.setCachePolicy(Config.VIndexCache) // Specify cache policy: NoCache / VIndexCache / BufferCache
|
||||||
.setSearchers(15) // 设置初始化的查询器数量
|
.setSearchers(15) // Set the number of initialized searchers
|
||||||
// .setXdbInputStream(InputStream) // 设置 v4 xdb 文件的 inputstream 对象
|
// .setCacheSliceBytes(int) // Set cache slice bytes, default is 50MiB
|
||||||
// .setXdbFile(File) // 设置 v4 xdb File 对象
|
// .setXdbInputStream(InputStream) // Set v4 xdb file inputstream object
|
||||||
.setXdbPath("ip2region v4 xdb path") // 设置 v4 xdb 文件的路径
|
// .setXdbFile(File) // Set v4 xdb File object
|
||||||
.asV4(); // 指定为 v4 配置
|
// .setFairLock(boolean) // Set whether ReentrantLock uses a fair lock
|
||||||
|
.setXdbPath("ip2region v4 xdb path") // Set the path of v4 xdb file
|
||||||
|
.asV4(); // Specify as v4 configuration
|
||||||
|
|
||||||
// 2, 创建 v6 的配置:指定缓存策略和 v6 的 xdb 文件路径
|
// 2. Create v6 configuration: specify cache policy and v6 xdb file path
|
||||||
final Config v6Config = Config.custom()
|
final Config v6Config = Config.custom()
|
||||||
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
.setCachePolicy(Config.VIndexCache) // Specify cache policy: NoCache / VIndexCache / BufferCache
|
||||||
.setSearchers(15) // 设置初始化的查询器数量
|
.setSearchers(15) // Set the number of initialized searchers
|
||||||
// .setXdbInputStream(InputStream) // 设置 v6 xdb 文件的 inputstream 对象
|
// .setCacheSliceBytes(int) // Set cache slice bytes, default is 50MiB
|
||||||
// .setXdbFile(File) // 设置 v6 xdb File 对象
|
// .setXdbInputStream(InputStream) // Set v6 xdb file inputstream object
|
||||||
.setXdbPath("ip2region v6 xdb path") // 设置 v6 xdb 文件的路径
|
// .setXdbFile(File) // Set v6 xdb File object
|
||||||
.asV6(); // 指定为 v6 配置
|
// .setFairLock(boolean) // Set whether ReentrantLock uses a fair lock
|
||||||
|
.setXdbPath("ip2region v6 xdb path") // Set the path of v6 xdb file
|
||||||
|
.asV6(); // Specify as v6 configuration
|
||||||
|
|
||||||
// 备注:Xdb 三种初始化输入的优先级:XdbInputStream -> XdbFile -> XdbPath
|
// Note: Priority for the three types of Xdb initialization inputs: XdbInputStream -> XdbFile -> XdbPath
|
||||||
// setXdbInputStream 仅方便使用者从 jar 包中加载 xdb 文件内容,这时 cachePolicy 只能设置为 Config.BufferCache
|
// setXdbInputStream is only for the convenience of users to load xdb file content from jar packages, in which case cachePolicy can only be set to Config.BufferCache
|
||||||
|
|
||||||
// 3,通过上述配置创建 Ip2Region 查询服务
|
// 3. Create Ip2Region query service through the above configurations
|
||||||
final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config);
|
final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config);
|
||||||
|
|
||||||
// 4,导出 ip2region 服务进行双版本的IP地址的并发查询,例如:
|
// 4. Export the ip2region service as a global variable to perform concurrent queries for both versions of IP addresses, for example:
|
||||||
final String v4Region = ip2Region.search("113.92.157.29"); // 进行 IPv4 查询
|
final String v4Region = ip2Region.search("113.92.157.29"); // Perform IPv4 query
|
||||||
final String v6Region = ip2Region.search("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); // 进行 IPv6 查询
|
final String v6Region = ip2Region.search("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); // Perform IPv6 query
|
||||||
|
|
||||||
// 5,在服务需要关闭的时候,同时关闭 ip2region 查询服务
|
// 5. When the service needs to be shut down, close the ip2region query service at the same time
|
||||||
|
// Note: The close method only needs to be called when the entire service is shut down; no operation is needed during queries
|
||||||
ip2Region.close();
|
ip2Region.close();
|
||||||
```
|
```
|
||||||
##### `Ip2Region` 查询备注:
|
|
||||||
1. 该查询服务的 API 并发安全且同时支持 `IPv4` 和 `IPv6` 的地址,内部实现会自动判断。
|
|
||||||
2. v4 和 v6 的配置需要单独创建,可以给 v4 和 v6 设置使用不同的缓存策略,也可以指定其中一个为 `null` 则该版本的 IP 地址查询都会返回 `null`。
|
|
||||||
3. 请结合您项目的并发数给 `setSearchers` 一个合适的查询器数量,默认为 20 个,这个值在运行过程中是固定的,每次查询会从池子里租借一个查询器来完成查询操作,查询完成后再归还回去,如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务,租借的锁是使用的 `ReentrantLock` 来管理,也可以通过如下方式来设置 `Ip2Region` 查询服务使用公平锁:
|
|
||||||
```java
|
|
||||||
final Ip2Region ip2region = Ip2Region.create(v4Config, v6Config, true);
|
|
||||||
```
|
|
||||||
4. 如果配置设置的缓存策略为 `Config.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器,该实现天生并发安全,此时通过 `setSearchers` 指定的查询器数量无效。
|
|
||||||
5. 如果 `ip2region` 查询器在提供服务期间,调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
|
|
||||||
|
|
||||||
|
##### `Ip2Region` Query Notes:
|
||||||
|
|
||||||
|
1. The API of this query service is concurrency-safe and supports both `IPv4` and `IPv6` addresses; the internal implementation will automatically distinguish them.
|
||||||
|
2. v4 and v6 configurations need to be created separately. You can set different cache policies for v4 and v6, or specify one of them as `null`, in which case IP address queries for that version will return `null`.
|
||||||
|
3. Please set a suitable number of searchers for `setSearchers` based on your project's concurrency. The default is 20. This value is fixed during runtime. Each query will borrow a searcher from the pool and return it after the query is completed. If the pool is empty when borrowing, it will wait until a searcher becomes available. The borrow lock is managed using `ReentrantLock`. You can also call `ConfigBuilder.setFairLock(true)` to force the service to use a fair lock.
|
||||||
|
4. If the cache policy in the configuration is set to `Config.BufferCache` (i.e., `Full Memory Cache`), a single-instance memory searcher will be used by default. This implementation is natively concurrency-safe, and the number of searchers specified via `setSearchers` will be ignored.
|
||||||
|
5. If `close` is called while the `ip2region` searcher is providing service, it will wait for a maximum of 10 seconds by default to allow as many searchers as possible to be returned.
|
||||||
|
|
||||||
|
### About Query API
|
||||||
|
|
||||||
|
The prototype of the location information query API is:
|
||||||
|
|
||||||
### 关于查询 API
|
|
||||||
定位信息查询 API 的原型为:
|
|
||||||
```java
|
```java
|
||||||
String search(String ipStr) throw Exception;
|
String search(String ipStr) throw Exception;
|
||||||
String search(byte[] ip) throw Exception;
|
String search(byte[] ip) throw Exception;
|
||||||
```
|
```
|
||||||
查询出错会抛出异常,如果查询成功会返回字符串的 `region` 信息,如果指定的 ip 查询不到会返回空字符串 `""`,这对于自定义数据或者数据不完整的情况会出现。
|
|
||||||
|
|
||||||
### 关于 IPv4 和 IPv6
|
An exception will be thrown if the query fails. If the query is successful, the `region` information string will be returned. If the specified IP cannot be found, an empty string `""` will be returned, which may occur for custom data or incomplete data.
|
||||||
该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询,使用方式如下:
|
|
||||||
|
### About IPv4 and IPv6
|
||||||
|
|
||||||
|
This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows:
|
||||||
|
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Version;
|
import org.lionsoul.ip2region.xdb.Version;
|
||||||
|
|
||||||
// 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件,IP版本指定为 Version.IPv4
|
// For IPv4: Set xdb path to the v4 xdb file, specify IP version as Version.IPv4
|
||||||
final String dbPath = "../../data/ip2region_v4.xdb"; // 或者你的 ipv4 xdb 的路径
|
final String dbPath = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path
|
||||||
final Version version = Version.IPv4;
|
final Version version = Version.IPv4;
|
||||||
|
|
||||||
// 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件,IP版本指定为 Version.IPv6
|
// For IPv6: Set xdb path to the v6 xdb file, specify IP version as Version.IPv6
|
||||||
final String dbPath = "../../data/ip2region_v6.xdb"; // 或者你的 ipv6 xdb 路径
|
final String dbPath = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path
|
||||||
final Version version = Version.IPv6;
|
final Version version = Version.IPv6;
|
||||||
|
|
||||||
// dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致,不然查询执行的时候会报错
|
// The IP version of the xdb specified by dbPath must be consistent with version, otherwise an error will occur during query execution
|
||||||
// 备注:以下演示直接使用 dbPath 和 version 变量
|
// Note: The following demonstration directly uses the dbPath and version variables
|
||||||
```
|
```
|
||||||
|
|
||||||
### 文件验证
|
### File Verification
|
||||||
建议您主动去验证 xdb 文件的适用性,因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件,验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证,例如在服务启动的时候,或者手动调用命令验证确认版本匹配即可,不要在每次创建的 Searcher 的时候运行验证,这样会影响查询的响应速度,尤其是高并发的使用场景。
|
|
||||||
|
It is recommended that you proactively verify the applicability of the xdb file, as some future new features may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify when the service starts or manually call a command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
```java
|
```java
|
||||||
try {
|
try {
|
||||||
Searcher.verifyFromFile(dbPath);
|
Searcher.verifyFromFile(dbPath);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// 适用性验证失败!!!
|
// Applicability verification failed!!!
|
||||||
// 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询.
|
// The current query client implementation is not suitable for querying the xdb file specified by dbPath.
|
||||||
// 应该停止启动服务,使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。
|
// You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with dbPath.
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 验证通过,当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作
|
// Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by dbPath
|
||||||
```
|
```
|
||||||
|
|
||||||
### 完全基于文件的查询
|
### File-Based Query
|
||||||
|
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
|
@ -108,7 +123,7 @@ import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearcherTest {
|
public class SearcherTest {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
// 1、使用上述的 version 和 dbPath 创建 searcher 对象
|
// 1. Create a searcher object using the version and dbPath mentioned above
|
||||||
Searcher searcher = null;
|
Searcher searcher = null;
|
||||||
try {
|
try {
|
||||||
searcher = Searcher.newWithFileOnly(version, dbPath);
|
searcher = Searcher.newWithFileOnly(version, dbPath);
|
||||||
|
|
@ -117,10 +132,10 @@ public class SearcherTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2、查询,IPv4 或者 IPv6 的地址都支持
|
// 2. Query, both IPv4 and IPv6 addresses are supported
|
||||||
try {
|
try {
|
||||||
String ip = "1.2.3.4";
|
String ip = "1.2.3.4";
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
long sTime = System.nanoTime();
|
long sTime = System.nanoTime();
|
||||||
String region = searcher.search(ip);
|
String region = searcher.search(ip);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
|
|
@ -129,17 +144,18 @@ public class SearcherTest {
|
||||||
System.out.printf("failed to search(%s): %s\n", ip, e);
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3、关闭资源
|
// 3. Close resources
|
||||||
searcher.close();
|
searcher.close();
|
||||||
|
|
||||||
// 备注:并发使用,每个线程需要创建一个独立的 searcher 对象单独使用。
|
// Note: For concurrent use, each thread needs to create an independent searcher object for separate use.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存 `VectorIndex` 索引
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
We can pre-load `VectorIndex` data from the `xdb` file and cache it globally. Using the global VectorIndex cache every time a Searcher object is created can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure.
|
||||||
|
|
||||||
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据,然后全局缓存,每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作,从而加速查询,减少 IO 压力。
|
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
|
|
@ -147,9 +163,9 @@ import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearcherTest {
|
public class SearcherTest {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
// 备注:version 和 dbPath 来源,请看上面的版本描述
|
// Note: For version and dbPath sources, please see the version description above
|
||||||
|
|
||||||
// 1、从 dbPath 中预先加载 VectorIndex 缓存,并且把这个得到的数据作为全局变量,后续反复使用。
|
// 1. Pre-load VectorIndex cache from dbPath and use the obtained data as a global variable for subsequent repeated use.
|
||||||
byte[] vIndex;
|
byte[] vIndex;
|
||||||
try {
|
try {
|
||||||
vIndex = Searcher.loadVectorIndexFromFile(dbPath);
|
vIndex = Searcher.loadVectorIndexFromFile(dbPath);
|
||||||
|
|
@ -158,7 +174,7 @@ public class SearcherTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
// 2. Use the global vIndex to create a query object with VectorIndex cache.
|
||||||
Searcher searcher;
|
Searcher searcher;
|
||||||
try {
|
try {
|
||||||
searcher = Searcher.newWithVectorIndex(version, dbPath, vIndex);
|
searcher = Searcher.newWithVectorIndex(version, dbPath, vIndex);
|
||||||
|
|
@ -167,10 +183,10 @@ public class SearcherTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3、查询,IPv4 或者 IPv6 地址都支持
|
// 3. Query, both IPv4 and IPv6 addresses are supported
|
||||||
try {
|
try {
|
||||||
String ip = "1.2.3.4";
|
String ip = "1.2.3.4";
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
long sTime = System.nanoTime();
|
long sTime = System.nanoTime();
|
||||||
String region = searcher.search(ip);
|
String region = searcher.search(ip);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
|
|
@ -179,17 +195,18 @@ public class SearcherTest {
|
||||||
System.out.printf("failed to search(%s): %s\n", ip, e);
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4、关闭资源
|
// 4. Close resources
|
||||||
searcher.close();
|
searcher.close();
|
||||||
|
|
||||||
// 备注:每个线程需要单独创建一个独立的 Searcher 对象,但是都共享全局的只读 vIndex 缓存。
|
// Note: Each thread needs to create an independent Searcher object, but they all share the same read-only global vIndex cache.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### Caching the Entire `xdb` File
|
||||||
|
|
||||||
|
We can also pre-load the entire xdb file data into memory and then create a query object based on this data to achieve a fully memory-based query, similar to the previous memory search.
|
||||||
|
|
||||||
我们也可以预先加载整个 xdb 文件的数据到内存,然后基于这个数据创建查询对象来实现完全基于文件的查询,类似之前的 memory search。
|
|
||||||
```java
|
```java
|
||||||
import org.lionsoul.ip2region.xdb.Searcher;
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
|
|
@ -197,10 +214,10 @@ import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
public class SearcherTest {
|
public class SearcherTest {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
// 备注:version 和 dbPath 来源,请看上面的版本描述
|
// Note: For version and dbPath sources, please see the version description above
|
||||||
|
|
||||||
// 1、从 dbPath 加载整个 xdb 到内存。
|
// 1. Load the entire xdb from dbPath into memory.
|
||||||
// 从这个 release 版本开始,xdb 的 buffer 使用 LongByteArray 来存储,避免 xdb 文件过大的时候 int 类型的溢出
|
// Starting from this release version, the xdb buffer uses LongByteArray for storage to avoid int type overflow when the xdb file is too large
|
||||||
LongByteArray cBuff;
|
LongByteArray cBuff;
|
||||||
try {
|
try {
|
||||||
cBuff = Searcher.loadContentFromFile(dbPath);
|
cBuff = Searcher.loadContentFromFile(dbPath);
|
||||||
|
|
@ -209,7 +226,7 @@ public class SearcherTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。
|
// 2. Use the above cBuff to create a fully memory-based query object.
|
||||||
Searcher searcher;
|
Searcher searcher;
|
||||||
try {
|
try {
|
||||||
searcher = Searcher.newWithBuffer(version, cBuff);
|
searcher = Searcher.newWithBuffer(version, cBuff);
|
||||||
|
|
@ -218,10 +235,10 @@ public class SearcherTest {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3、查询,IPv4 和 IPv6 都支持
|
// 3. Query, both IPv4 and IPv6 are supported
|
||||||
try {
|
try {
|
||||||
String ip = "1.2.3.4";
|
String ip = "1.2.3.4";
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
long sTime = System.nanoTime();
|
long sTime = System.nanoTime();
|
||||||
String region = searcher.search(ip);
|
String region = searcher.search(ip);
|
||||||
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
|
|
@ -230,43 +247,44 @@ public class SearcherTest {
|
||||||
System.out.printf("failed to search(%s): %s\n", ip, e);
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4、关闭资源 - 该 searcher 对象可以安全用于并发,等整个服务关闭的时候再关闭 searcher
|
// 4. Close resources - this searcher object can be safely used for concurrency; close it when the entire service is shut down
|
||||||
// searcher.close();
|
// searcher.close();
|
||||||
|
|
||||||
// 备注:并发使用,用整个 xdb 数据缓存创建的查询对象可以安全的用于并发,也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。
|
// Note: For concurrent use, the query object created with the entire xdb data cache can be safely used for concurrency,
|
||||||
|
// meaning you can make this searcher object a global object for cross-thread access.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
如果调用 `loadContentXXX` 方法来加载 xdb buffer 的过程中出现了 OOM 错误,请参考以下的 [sliceBytes 设置](#slicebytes),选择使用带 sliceBytes 参数的 `loadContentXXX` 方法来加载 。
|
If an OOM error occurs while calling the `loadContentXXX` method to load the xdb buffer, please refer to the [sliceBytes setting](#slicebytes) below and choose the `loadContentXXX` method with the sliceBytes parameter.
|
||||||
|
|
||||||
### sliceBytes
|
### sliceBytes
|
||||||
|
|
||||||
sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List<byte[]> buffs` 集合的分片内存的大小,默认值为 `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`,这个值的最大允许值为 `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`,关于该取值的来源可以参考作者博客文章:[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
|
sliceBytes represents the size of the partitioned memory for the `List<byte[]> buffs` collection maintained inside the `LongByteArray` class during full memory caching. The default value is `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`. The maximum allowed value is `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`. For the source of this value, please refer to the author's blog post: [https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA).
|
||||||
1. 从 `3.3.3` 版本开始 `LongByteArray` 实现了固定分片尺寸支持,可以通过简单的计算快速的完成 `offset` 定位的从而实现 `slice` 或者 `copy` 操作。
|
|
||||||
2. 从计算速度来说 sliceBytes 越大 buffs 的长度越小,计算耗时越小,不过有了固定 sliceBytes 实现这个差距完全可以忽略,所以建议保持默认值为 `50MiB` 即可,也不会出现之前弹性分片尺寸可能导致的 OOM 问题。
|
|
||||||
|
|
||||||
|
1. Starting from version `3.3.3`, `LongByteArray` implements fixed partition size support, which allows for fast `offset` positioning through simple calculation to perform `slice` or `copy` operations.
|
||||||
|
2. In terms of calculation speed, the larger the sliceBytes, the smaller the length of buffs and the lower the calculation time. However, with the fixed sliceBytes implementation, this gap is completely negligible. Therefore, it is recommended to keep the default value of `50MiB`, which also avoids the OOM issues that could be caused by elastic partition sizes previously.
|
||||||
|
|
||||||
|
# Compiling the Test Program
|
||||||
|
|
||||||
# 编译测试程序
|
Compile the test program via Maven.
|
||||||
|
|
||||||
通过 maven 来编译测试程序。
|
|
||||||
```bash
|
```bash
|
||||||
# cd 到 java binding 的根目录
|
# cd to the root directory of java binding
|
||||||
cd binding/java/
|
cd binding/java/
|
||||||
mvn compile package
|
mvn compile package
|
||||||
```
|
```
|
||||||
|
|
||||||
然后会在当前目录的 target 目录下得到一个 ip2region-{version}.jar 的打包文件。
|
Then a packaged file named ip2region-{version}.jar will be generated in the target directory under the current folder.
|
||||||
|
|
||||||
|
# Query Testing
|
||||||
|
|
||||||
|
### Test Command
|
||||||
|
|
||||||
# 查询测试
|
You can test queries via the `java -jar target/ip2region-{version}.jar search` command:
|
||||||
|
|
||||||
### 测试命令
|
|
||||||
可以通过 `java -jar target/ip2region-{version}.jar search` 命令来测试查询:
|
|
||||||
```bash
|
```bash
|
||||||
➜ java git:(master) ✗ java -jar target/ip2region-3.3.0.jar search --help
|
➜ java git:(master) ✗ java -jar target/ip2region-3.3.4.jar search --help
|
||||||
java -jar ip2region-{version}.jar search [command options]
|
java -jar ip2region-{version}.jar search [command options]
|
||||||
options:
|
options:
|
||||||
--v4-db string ip2region ipv4 binary xdb file path
|
--v4-db string ip2region ipv4 binary xdb file path
|
||||||
|
|
@ -276,35 +294,41 @@ options:
|
||||||
--help print this help menu
|
--help print this help menu
|
||||||
```
|
```
|
||||||
|
|
||||||
### 参数解析
|
### Parameter Parsing
|
||||||
1. `v4-xdb`: IPv4 的 xdb 文件路径,默认为仓库中的 data/ip2region_v4.xdb
|
|
||||||
2. `v6-xdb`: IPv6 的 xdb 文件路径,默认为仓库中的 data/ip2region_v6.xdb
|
1. `v4-xdb`: IPv4 xdb file path, defaults to data/ip2region_v4.xdb in the repository.
|
||||||
3. `v4-cache-policy`: v4 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
2. `v6-xdb`: IPv6 xdb file path, defaults to data/ip2region_v6.xdb in the repository.
|
||||||
4. `v6-cache-policy`: v6 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
3. `v4-cache-policy`: Cache policy used for v4 queries, default is `vectorIndex`, options: file/vectorIndex/content.
|
||||||
|
4. `v6-cache-policy`: Cache policy used for v6 queries, default is `vectorIndex`, options: file/vectorIndex/content.
|
||||||
|
|
||||||
|
### Test Demo
|
||||||
|
|
||||||
|
Example: performing query testing using default data/ip2region_v4.xdb and data/ip2region_v6.xdb:
|
||||||
|
|
||||||
### 测试 Demo
|
|
||||||
例如:使用默认的 data/ip2region_v4.xdb 和 data/ip2region_v6.xdb 进行查询测试:
|
|
||||||
```bash
|
```bash
|
||||||
➜ java git:(java_app_with_ip2region_service) ✗ java -jar target/ip2region-3.3.0.jar search
|
➜ java git:(java_app_with_ip2region_service) ✗ java -jar target/ip2region-3.3.4.jar search
|
||||||
ip2region search service test program
|
ip2region search service test program
|
||||||
+-v4 xdb: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex)
|
+-v4 xdb: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex)
|
||||||
+-v6 xdb: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex)
|
+-v6 xdb: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
ip2region>> 1.2.3.4
|
ip2region>> 1.2.3.4
|
||||||
{region: 美国|华盛顿|0|谷歌, took: 159 μs}
|
{region: Australia|Queensland|Brisbane|0|AU, took: 140 μs}
|
||||||
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, took: 346 μs}
|
{region: 中国|广东省|深圳市|电信|CN, took: 391 μs}
|
||||||
ip2region>>
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, took: 503 μs}
|
||||||
```
|
```
|
||||||
输入 v4 或者 v6 的 IP 地址即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
|
|
||||||
|
|
||||||
|
Enter a v4 or v6 IP address to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the effects of the three different cache implementations.
|
||||||
|
|
||||||
# bench 测试
|
# bench Testing
|
||||||
|
|
||||||
|
### Test Command
|
||||||
|
|
||||||
|
You can perform bench testing via the `java -jar ip2region-{version}.jar bench` command to ensure the `xdb` file is error-free and to evaluate query performance:
|
||||||
|
|
||||||
### 测试命令
|
|
||||||
可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试,一方面确保 `xdb` 文件没有错误,一方面可以评估查询性能:
|
|
||||||
```bash
|
```bash
|
||||||
➜ java git:(fr_java_ipv6) ✗ java -jar target/ip2region-3.1.0.jar bench
|
➜ java git:(fr_java_ipv6) ✗ java -jar target/ip2region-3.3.4.jar bench
|
||||||
java -jar ip2region-{version}.jar bench [command options]
|
java -jar ip2region-{version}.jar bench [command options]
|
||||||
options:
|
options:
|
||||||
--db string ip2region binary xdb file path
|
--db string ip2region binary xdb file path
|
||||||
|
|
@ -313,16 +337,20 @@ options:
|
||||||
```
|
```
|
||||||
|
|
||||||
### v4 bench
|
### v4 bench
|
||||||
例如:通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试:
|
|
||||||
|
Example: IPv4 bench testing using default data/ip2region_v4.xdb and data/ipv4_source.txt files:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
java -jar target/ip2region-3.1.0.jar bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
java -jar target/ip2region-3.3.4.jar bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
### v6 bench
|
### v6 bench
|
||||||
例如:通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试:
|
|
||||||
|
Example: IPv6 bench testing using default data/ip2region_v6.xdb and data/ipv6_source.txt files:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
java -jar target/ip2region-3.1.0.jar bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
java -jar target/ip2region-3.3.4.jar bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。
|
You can test the effects of the three different cache implementations by setting `cache-policy` to file/vectorIndex/content.
|
||||||
@Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。
|
@Note: Please ensure that the src file used for benching is the same source file used to generate the corresponding xdb file.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,333 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region java 查询客户端
|
||||||
|
|
||||||
|
# 使用方式
|
||||||
|
|
||||||
|
### maven 仓库:
|
||||||
|
```xml
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.lionsoul</groupId>
|
||||||
|
<artifactId>ip2region</artifactId>
|
||||||
|
<version>3.3.7</version>
|
||||||
|
</dependency>
|
||||||
|
```
|
||||||
|
|
||||||
|
### 关于查询服务
|
||||||
|
从 `3.2.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务,**建议优先使用该方式来进行查询调用**,具体使用方式如下:
|
||||||
|
```java
|
||||||
|
import org.lionsoul.ip2region.service.Config;
|
||||||
|
import org.lionsoul.ip2region.service.Ip2Region;
|
||||||
|
|
||||||
|
// 1, 创建 v4 的配置:指定缓存策略和 v4 的 xdb 文件路径
|
||||||
|
final Config v4Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
||||||
|
.setSearchers(15) // 设置初始化的查询器数量
|
||||||
|
// .setCacheSliceBytes(int) // 设置缓存的分片字节数,默认为 50MiB
|
||||||
|
// .setXdbInputStream(InputStream) // 设置 v4 xdb 文件的 inputstream 对象
|
||||||
|
// .setXdbFile(File) // 设置 v4 xdb File 对象
|
||||||
|
// .setFairLock(boolean) // 设置 ReentrantLock 是否使用公平锁
|
||||||
|
.setXdbPath("ip2region v4 xdb path") // 设置 v4 xdb 文件的路径
|
||||||
|
.asV4(); // 指定为 v4 配置
|
||||||
|
|
||||||
|
// 2, 创建 v6 的配置:指定缓存策略和 v6 的 xdb 文件路径
|
||||||
|
final Config v6Config = Config.custom()
|
||||||
|
.setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache
|
||||||
|
.setSearchers(15) // 设置初始化的查询器数量
|
||||||
|
// .setCacheSliceBytes(int) // 设置缓存的分片字节数,默认为 50MiB
|
||||||
|
// .setXdbInputStream(InputStream) // 设置 v6 xdb 文件的 inputstream 对象
|
||||||
|
// .setXdbFile(File) // 设置 v6 xdb File 对象
|
||||||
|
// .setFairLock(boolean) // 设置 ReentrantLock 是否使用公平锁
|
||||||
|
.setXdbPath("ip2region v6 xdb path") // 设置 v6 xdb 文件的路径
|
||||||
|
.asV6(); // 指定为 v6 配置
|
||||||
|
|
||||||
|
// 备注:Xdb 三种初始化输入的优先级:XdbInputStream -> XdbFile -> XdbPath
|
||||||
|
// setXdbInputStream 仅方便使用者从 jar 包中加载 xdb 文件内容,这时 cachePolicy 只能设置为 Config.BufferCache
|
||||||
|
|
||||||
|
// 3,通过上述配置创建 Ip2Region 查询服务
|
||||||
|
final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config);
|
||||||
|
|
||||||
|
// 4,导出 ip2region 服务作为全局变量,进行双版本的IP地址的并发查询,例如:
|
||||||
|
final String v4Region = ip2Region.search("113.92.157.29"); // 进行 IPv4 查询
|
||||||
|
final String v6Region = ip2Region.search("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); // 进行 IPv6 查询
|
||||||
|
|
||||||
|
// 5,在服务需要关闭的时候,同时关闭 ip2region 查询服务
|
||||||
|
// 备注:close 方法只需要在整个服务关闭的时候关闭,查询途中不需要操作
|
||||||
|
ip2Region.close();
|
||||||
|
```
|
||||||
|
##### `Ip2Region` 查询备注:
|
||||||
|
1. 该查询服务的 API 并发安全且同时支持 `IPv4` 和 `IPv6` 的地址,内部实现会自动判断。
|
||||||
|
2. v4 和 v6 的配置需要单独创建,可以给 v4 和 v6 设置使用不同的缓存策略,也可以指定其中一个为 `null` 则该版本的 IP 地址查询都会返回 `null`。
|
||||||
|
3. 请结合您项目的并发数给 `setSearchers` 一个合适的查询器数量,默认为 20 个,这个值在运行过程中是固定的,每次查询会从池子里租借一个查询器来完成查询操作,查询完成后再归还回去,如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务,租借的锁是使用的 `ReentrantLock` 来管理,也可以通过调用 `ConfigBuilder.setFairLock(true)` 来配置查询服务使用公平锁:
|
||||||
|
4. 如果配置设置的缓存策略为 `Config.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器,该实现天生并发安全,此时通过 `setSearchers` 指定的查询器数量无效。
|
||||||
|
5. 如果 `ip2region` 查询器在提供服务期间,调用 close 默认会最大等待 10 秒钟来等待尽量多的查询器归还。
|
||||||
|
|
||||||
|
|
||||||
|
### 关于查询 API
|
||||||
|
定位信息查询 API 的原型为:
|
||||||
|
```java
|
||||||
|
String search(String ipStr) throw Exception;
|
||||||
|
String search(byte[] ip) throw Exception;
|
||||||
|
```
|
||||||
|
查询出错会抛出异常,如果查询成功会返回字符串的 `region` 信息,如果指定的 ip 查询不到会返回空字符串 `""`,这对于自定义数据或者数据不完整的情况会出现。
|
||||||
|
|
||||||
|
### 关于 IPv4 和 IPv6
|
||||||
|
该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询,使用方式如下:
|
||||||
|
```java
|
||||||
|
import org.lionsoul.ip2region.xdb.Version;
|
||||||
|
|
||||||
|
// 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件,IP版本指定为 Version.IPv4
|
||||||
|
final String dbPath = "../../data/ip2region_v4.xdb"; // 或者你的 ipv4 xdb 的路径
|
||||||
|
final Version version = Version.IPv4;
|
||||||
|
|
||||||
|
// 如果是 IPv6: 设置 xdb 路径为 v6 的 xdb 文件,IP版本指定为 Version.IPv6
|
||||||
|
final String dbPath = "../../data/ip2region_v6.xdb"; // 或者你的 ipv6 xdb 路径
|
||||||
|
final Version version = Version.IPv6;
|
||||||
|
|
||||||
|
// dbPath 指定的 xdb 的 IP 版本必须和 version 指定的一致,不然查询执行的时候会报错
|
||||||
|
// 备注:以下演示直接使用 dbPath 和 version 变量
|
||||||
|
```
|
||||||
|
|
||||||
|
### 文件验证
|
||||||
|
建议您主动去验证 xdb 文件的适用性,因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件,验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证,例如在服务启动的时候,或者手动调用命令验证确认版本匹配即可,不要在每次创建的 Searcher 的时候运行验证,这样会影响查询的响应速度,尤其是高并发的使用场景。
|
||||||
|
```java
|
||||||
|
try {
|
||||||
|
Searcher.verifyFromFile(dbPath);
|
||||||
|
} catch (Exception e) {
|
||||||
|
// 适用性验证失败!!!
|
||||||
|
// 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询.
|
||||||
|
// 应该停止启动服务,使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 验证通过,当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作
|
||||||
|
```
|
||||||
|
|
||||||
|
### 完全基于文件的查询
|
||||||
|
|
||||||
|
```java
|
||||||
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
import java.io.*;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public class SearcherTest {
|
||||||
|
public static void main(String[] args) {
|
||||||
|
// 1、使用上述的 version 和 dbPath 创建 searcher 对象
|
||||||
|
Searcher searcher = null;
|
||||||
|
try {
|
||||||
|
searcher = Searcher.newWithFileOnly(version, dbPath);
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2、查询,IPv4 或者 IPv6 的地址都支持
|
||||||
|
try {
|
||||||
|
String ip = "1.2.3.4";
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
long sTime = System.nanoTime();
|
||||||
|
String region = searcher.search(ip);
|
||||||
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3、关闭资源
|
||||||
|
searcher.close();
|
||||||
|
|
||||||
|
// 备注:并发使用,每个线程需要创建一个独立的 searcher 对象单独使用。
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 缓存 `VectorIndex` 索引
|
||||||
|
|
||||||
|
我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据,然后全局缓存,每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作,从而加速查询,减少 IO 压力。
|
||||||
|
```java
|
||||||
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
import java.io.*;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public class SearcherTest {
|
||||||
|
public static void main(String[] args) {
|
||||||
|
// 备注:version 和 dbPath 来源,请看上面的版本描述
|
||||||
|
|
||||||
|
// 1、从 dbPath 中预先加载 VectorIndex 缓存,并且把这个得到的数据作为全局变量,后续反复使用。
|
||||||
|
byte[] vIndex;
|
||||||
|
try {
|
||||||
|
vIndex = Searcher.loadVectorIndexFromFile(dbPath);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to load vector index from `%s`: %s\n", dbPath, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
||||||
|
Searcher searcher;
|
||||||
|
try {
|
||||||
|
searcher = Searcher.newWithVectorIndex(version, dbPath, vIndex);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3、查询,IPv4 或者 IPv6 地址都支持
|
||||||
|
try {
|
||||||
|
String ip = "1.2.3.4";
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
long sTime = System.nanoTime();
|
||||||
|
String region = searcher.search(ip);
|
||||||
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4、关闭资源
|
||||||
|
searcher.close();
|
||||||
|
|
||||||
|
// 备注:每个线程需要单独创建一个独立的 Searcher 对象,但是都共享全局的只读 vIndex 缓存。
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
|
我们也可以预先加载整个 xdb 文件的数据到内存,然后基于这个数据创建查询对象来实现完全基于文件的查询,类似之前的 memory search。
|
||||||
|
```java
|
||||||
|
import org.lionsoul.ip2region.xdb.Searcher;
|
||||||
|
import java.io.*;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
public class SearcherTest {
|
||||||
|
public static void main(String[] args) {
|
||||||
|
// 备注:version 和 dbPath 来源,请看上面的版本描述
|
||||||
|
|
||||||
|
// 1、从 dbPath 加载整个 xdb 到内存。
|
||||||
|
// 从这个 release 版本开始,xdb 的 buffer 使用 LongByteArray 来存储,避免 xdb 文件过大的时候 int 类型的溢出
|
||||||
|
LongByteArray cBuff;
|
||||||
|
try {
|
||||||
|
cBuff = Searcher.loadContentFromFile(dbPath);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to load content from `%s`: %s\n", dbPath, e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2、使用上述的 cBuff 创建一个完全基于内存的查询对象。
|
||||||
|
Searcher searcher;
|
||||||
|
try {
|
||||||
|
searcher = Searcher.newWithBuffer(version, cBuff);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to create content cached searcher: %s\n", e);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3、查询,IPv4 和 IPv6 都支持
|
||||||
|
try {
|
||||||
|
String ip = "1.2.3.4";
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
long sTime = System.nanoTime();
|
||||||
|
String region = searcher.search(ip);
|
||||||
|
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
|
||||||
|
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.printf("failed to search(%s): %s\n", ip, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4、关闭资源 - 该 searcher 对象可以安全用于并发,等整个服务关闭的时候再关闭 searcher
|
||||||
|
// searcher.close();
|
||||||
|
|
||||||
|
// 备注:并发使用,用整个 xdb 数据缓存创建的查询对象可以安全的用于并发,也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
如果调用 `loadContentXXX` 方法来加载 xdb buffer 的过程中出现了 OOM 错误,请参考以下的 [sliceBytes 设置](#slicebytes),选择使用带 sliceBytes 参数的 `loadContentXXX` 方法来加载 。
|
||||||
|
|
||||||
|
### sliceBytes
|
||||||
|
|
||||||
|
sliceBytes 表示 xdb 全内存缓存时 `LongByteArray` 类内部维护的 `List<byte[]> buffs` 集合的分片内存的大小,默认值为 `Searcher.DEFAULT_SLICE_BYTES` = `50MiB`,这个值的最大允许值为 `Searcher.MAX_WRITE_BYTES` = `0x7ffff000`,关于该取值的来源可以参考作者博客文章:[https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA](https://mp.weixin.qq.com/s/4xHRcnQbIcjtMGdXEGrxsA)。
|
||||||
|
1. 从 `3.3.3` 版本开始 `LongByteArray` 实现了固定分片尺寸支持,可以通过简单的计算快速的完成 `offset` 定位的从而实现 `slice` 或者 `copy` 操作。
|
||||||
|
2. 从计算速度来说 sliceBytes 越大 buffs 的长度越小,计算耗时越小,不过有了固定 sliceBytes 实现这个差距完全可以忽略,所以建议保持默认值为 `50MiB` 即可,也不会出现之前弹性分片尺寸可能导致的 OOM 问题。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# 编译测试程序
|
||||||
|
|
||||||
|
通过 maven 来编译测试程序。
|
||||||
|
```bash
|
||||||
|
# cd 到 java binding 的根目录
|
||||||
|
cd binding/java/
|
||||||
|
mvn compile package
|
||||||
|
```
|
||||||
|
|
||||||
|
然后会在当前目录的 target 目录下得到一个 ip2region-{version}.jar 的打包文件。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# 查询测试
|
||||||
|
|
||||||
|
### 测试命令
|
||||||
|
可以通过 `java -jar target/ip2region-{version}.jar search` 命令来测试查询:
|
||||||
|
```bash
|
||||||
|
➜ java git:(master) ✗ java -jar target/ip2region-3.3.0.jar search --help
|
||||||
|
java -jar ip2region-{version}.jar search [command options]
|
||||||
|
options:
|
||||||
|
--v4-db string ip2region ipv4 binary xdb file path
|
||||||
|
--v4-cache-policy string v4 cache policy, default vectorIndex, options: file/vectorIndex/content
|
||||||
|
--v6-db string ip2region ipv6 binary xdb file path
|
||||||
|
--v6-cache-policy string v6 cache policy, default vectorIndex, options: file/vectorIndex/content
|
||||||
|
--help print this help menu
|
||||||
|
```
|
||||||
|
|
||||||
|
### 参数解析
|
||||||
|
1. `v4-xdb`: IPv4 的 xdb 文件路径,默认为仓库中的 data/ip2region_v4.xdb
|
||||||
|
2. `v6-xdb`: IPv6 的 xdb 文件路径,默认为仓库中的 data/ip2region_v6.xdb
|
||||||
|
3. `v4-cache-policy`: v4 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
||||||
|
4. `v6-cache-policy`: v6 查询使用的缓存策略,默认为 `vectorIndex`,可选:file/vectorIndex/content
|
||||||
|
|
||||||
|
### 测试 Demo
|
||||||
|
例如:使用默认的 data/ip2region_v4.xdb 和 data/ip2region_v6.xdb 进行查询测试:
|
||||||
|
```bash
|
||||||
|
➜ java git:(java_app_with_ip2region_service) ✗ java -jar target/ip2region-3.3.0.jar search
|
||||||
|
ip2region search service test program
|
||||||
|
+-v4 xdb: /data01/code/c/ip2region/data/ip2region_v4.xdb (vectorIndex)
|
||||||
|
+-v6 xdb: /data01/code/c/ip2region/data/ip2region_v6.xdb (vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, took: 140 μs}
|
||||||
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, took: 391 μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, took: 503 μs}
|
||||||
|
```
|
||||||
|
输入 v4 或者 v6 的 IP 地址即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
|
||||||
|
|
||||||
|
|
||||||
|
# bench 测试
|
||||||
|
|
||||||
|
### 测试命令
|
||||||
|
可以通过 `java -jar ip2region-{version}.jar bench` 命令来进行 bench 测试,一方面确保 `xdb` 文件没有错误,一方面可以评估查询性能:
|
||||||
|
```bash
|
||||||
|
➜ java git:(fr_java_ipv6) ✗ java -jar target/ip2region-3.1.0.jar bench
|
||||||
|
java -jar ip2region-{version}.jar bench [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--src string source ip text file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
### v4 bench
|
||||||
|
例如:通过默认的 data/ip2region_v4.xdb 和 data/ipv4_source.txt 文件进行 IPv4 的 bench 测试:
|
||||||
|
```bash
|
||||||
|
java -jar target/ip2region-3.1.0.jar bench --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### v6 bench
|
||||||
|
例如:通过默认的 data/ip2region_v6.xdb 和 data/ipv6_source.txt 文件进行 IPv6 的 bench 测试:
|
||||||
|
```bash
|
||||||
|
java -jar target/ip2region-3.1.0.jar bench --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。
|
||||||
|
@Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。
|
||||||
|
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
<groupId>org.lionsoul</groupId>
|
<groupId>org.lionsoul</groupId>
|
||||||
<artifactId>ip2region</artifactId>
|
<artifactId>ip2region</artifactId>
|
||||||
<version>3.3.4</version>
|
<version>3.3.7</version>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
<name>ip2region</name>
|
<name>ip2region</name>
|
||||||
|
|
|
||||||
|
|
@ -39,13 +39,16 @@ public class Config {
|
||||||
|
|
||||||
public final int searchers;
|
public final int searchers;
|
||||||
|
|
||||||
|
// force ReenTrantLock to use fair Lock
|
||||||
|
public final boolean fairLock;
|
||||||
|
|
||||||
// config builder
|
// config builder
|
||||||
public static ConfigBuilder custom() {
|
public static ConfigBuilder custom() {
|
||||||
return new ConfigBuilder();
|
return new ConfigBuilder();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Config(int cachePolicy, Version ipVersion, File xdbFile,
|
protected Config(int cachePolicy, Version ipVersion, File xdbFile,
|
||||||
Header header, byte[] vIndex, LongByteArray cBuffer, int searchers) throws IOException, XdbException {
|
Header header, byte[] vIndex, LongByteArray cBuffer, int searchers, boolean fairLock) throws IOException, XdbException {
|
||||||
this.cachePolicy = cachePolicy;
|
this.cachePolicy = cachePolicy;
|
||||||
this.ipVersion = ipVersion;
|
this.ipVersion = ipVersion;
|
||||||
|
|
||||||
|
|
@ -62,6 +65,7 @@ public class Config {
|
||||||
}
|
}
|
||||||
|
|
||||||
this.searchers = searchers;
|
this.searchers = searchers;
|
||||||
|
this.fairLock = fairLock;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override public String toString() {
|
@Override public String toString() {
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,9 @@ public class ConfigBuilder {
|
||||||
// searchers
|
// searchers
|
||||||
private int searchers = 20;
|
private int searchers = 20;
|
||||||
|
|
||||||
|
// fair lock
|
||||||
|
private boolean fairLock = false;
|
||||||
|
|
||||||
public ConfigBuilder() {}
|
public ConfigBuilder() {}
|
||||||
|
|
||||||
public ConfigBuilder(String xdbPath) {
|
public ConfigBuilder(String xdbPath) {
|
||||||
|
|
@ -49,16 +52,19 @@ public class ConfigBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
public ConfigBuilder setXdbPath(String xdbPath) {
|
public ConfigBuilder setXdbPath(String xdbPath) {
|
||||||
|
assert xdbPath != null && xdbPath.length() > 0;
|
||||||
this.xdbPath = xdbPath;
|
this.xdbPath = xdbPath;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ConfigBuilder setXdbFile(File xdbFile) {
|
public ConfigBuilder setXdbFile(File xdbFile) {
|
||||||
|
assert xdbFile != null;
|
||||||
this.xdbFile = xdbFile;
|
this.xdbFile = xdbFile;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ConfigBuilder setXdbInputStream(InputStream xdbInputStream) {
|
public ConfigBuilder setXdbInputStream(InputStream xdbInputStream) {
|
||||||
|
assert xdbInputStream != null;
|
||||||
this.xdbInputStream = xdbInputStream;
|
this.xdbInputStream = xdbInputStream;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
@ -73,6 +79,11 @@ public class ConfigBuilder {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ConfigBuilder setFairLock(boolean fairLock) {
|
||||||
|
this.fairLock = fairLock;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
private Config build(Version ipVersion) throws IOException, XdbException, InvalidConfigException {
|
private Config build(Version ipVersion) throws IOException, XdbException, InvalidConfigException {
|
||||||
if (xdbInputStream == null) {
|
if (xdbInputStream == null) {
|
||||||
// everyting is fine
|
// everyting is fine
|
||||||
|
|
@ -84,14 +95,14 @@ public class ConfigBuilder {
|
||||||
// 1, load the content buffer
|
// 1, load the content buffer
|
||||||
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(xdbInputStream, cacheSliceBytes);
|
final LongByteArray cBuffer = Searcher.loadContentFromInputStream(xdbInputStream, cacheSliceBytes);
|
||||||
|
|
||||||
// 2, verify the xdb from the buffer
|
// 2, load the header
|
||||||
Searcher.verify(Searcher.loadHeaderFromBuffer(cBuffer), cBuffer.length());
|
|
||||||
|
|
||||||
// 3, load the header
|
|
||||||
final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
|
final Header header = Searcher.loadHeaderFromBuffer(cBuffer);
|
||||||
|
|
||||||
|
// 3, verify the xdb from the buffer
|
||||||
|
Searcher.verify(header, cBuffer.length());
|
||||||
|
|
||||||
// create the config without xdbFile and vIndex
|
// create the config without xdbFile and vIndex
|
||||||
return new Config(cachePolicy, ipVersion, null, header, null, cBuffer, searchers);
|
return new Config(cachePolicy, ipVersion, null, header, null, cBuffer, searchers, fairLock);
|
||||||
}
|
}
|
||||||
|
|
||||||
// load the header and the cache buffer
|
// load the header and the cache buffer
|
||||||
|
|
@ -119,7 +130,7 @@ public class ConfigBuilder {
|
||||||
final LongByteArray cBuffer = cachePolicy == Config.BufferCache ? Searcher.loadContent(raf, cacheSliceBytes) : null;
|
final LongByteArray cBuffer = cachePolicy == Config.BufferCache ? Searcher.loadContent(raf, cacheSliceBytes) : null;
|
||||||
|
|
||||||
raf.close();
|
raf.close();
|
||||||
return new Config(cachePolicy, ipVersion, xdbFile, header, vIndex, cBuffer, searchers);
|
return new Config(cachePolicy, ipVersion, xdbFile, header, vIndex, cBuffer, searchers, fairLock);
|
||||||
}
|
}
|
||||||
|
|
||||||
// build the final #Config instance for IPv4
|
// build the final #Config instance for IPv4
|
||||||
|
|
|
||||||
|
|
@ -81,7 +81,7 @@ public class Ip2Region {
|
||||||
this.v4Pool = null;
|
this.v4Pool = null;
|
||||||
} else {
|
} else {
|
||||||
this.v4InMemSearcher = null;
|
this.v4InMemSearcher = null;
|
||||||
this.v4Pool = new SearcherPool(v4Config);
|
this.v4Pool = new SearcherPool(v4Config, v4Config.fairLock);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (v6Config == null) {
|
if (v6Config == null) {
|
||||||
|
|
@ -93,7 +93,7 @@ public class Ip2Region {
|
||||||
this.v6Pool = null;
|
this.v6Pool = null;
|
||||||
} else {
|
} else {
|
||||||
this.v6InMemSearcher = null;
|
this.v6InMemSearcher = null;
|
||||||
this.v6Pool = new SearcherPool(v6Config);
|
this.v6Pool = new SearcherPool(v6Config, v6Config.fairLock);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -138,6 +138,11 @@ public class Searcher {
|
||||||
}
|
}
|
||||||
|
|
||||||
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
|
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
|
||||||
|
// @Note: ptr validate, zero ptr means source data missing
|
||||||
|
// so we could just stop here and return an empty string.
|
||||||
|
if (sPtr == 0 || ePtr == 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
// binary search the segment index block to get the region info
|
// binary search the segment index block to get the region info
|
||||||
final int bytes = ip.length, dBytes = ip.length << 1;
|
final int bytes = ip.length, dBytes = ip.length << 1;
|
||||||
|
|
|
||||||
|
|
@ -119,9 +119,9 @@ public class Ip2RegionTest {
|
||||||
try {
|
try {
|
||||||
final String region = ip2Region.search(ipBytes);
|
final String region = ip2Region.search(ipBytes);
|
||||||
if (ipBytes.length == 4) {
|
if (ipBytes.length == 4) {
|
||||||
assertEquals("v4 region not equals", region, "中国|广东省|深圳市|电信");
|
assertEquals("v4 region not equals", region, "中国|广东省|深圳市|电信|CN");
|
||||||
} else {
|
} else {
|
||||||
assertEquals("v6 region not equals", region, "中国|广东省|深圳市|家庭宽带");
|
assertEquals("v6 region not equals", region, "中国|广东省|深圳市|电信|CN");
|
||||||
}
|
}
|
||||||
} catch (InetAddressException | IOException | InterruptedException e) {
|
} catch (InetAddressException | IOException | InterruptedException e) {
|
||||||
log.errorf("failed to search(%s): %s", Util.ipToString(ipBytes), e.getMessage());
|
log.errorf("failed to search(%s): %s", Util.ipToString(ipBytes), e.getMessage());
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,257 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region JavaScript Query Client
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### Install `ip2region.js`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install ip2region.js --save
|
||||||
|
```
|
||||||
|
|
||||||
|
### About Query API
|
||||||
|
|
||||||
|
The prototype of the Query API is:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
// Query via a string IP or a binary IP (Buffer type) parsed by parseIP
|
||||||
|
search(ip: string | Buffer): string;
|
||||||
|
```
|
||||||
|
|
||||||
|
If an error occurs during the query, an exception will be thrown. If the query is successful, the `region` information string will be returned. If the specified IP cannot be found, an empty string `""` will be returned.
|
||||||
|
|
||||||
|
### About IPv4 and IPv6
|
||||||
|
|
||||||
|
This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import {IPv4, IPv6} from 'ip2region.js';
|
||||||
|
|
||||||
|
// For IPv4: Set xdb path to the v4 xdb file, and set IP version to Version.IPv4
|
||||||
|
let dbPath = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path
|
||||||
|
let version = IPv4;
|
||||||
|
|
||||||
|
// For IPv6: Set xdb path to the v6 xdb file, and set IP version to Version.IPv6
|
||||||
|
let dbPath = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path
|
||||||
|
let version = IPv6;
|
||||||
|
|
||||||
|
// The IP version of the xdb specified by dbPath must match the version specified; otherwise, an error will occur during execution.
|
||||||
|
// Note: The following demonstrations directly use the dbPath and version variables.
|
||||||
|
```
|
||||||
|
|
||||||
|
### File Verification
|
||||||
|
|
||||||
|
It is recommended that you proactively verify the applicability of the xdb file. Some new features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification helps avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify once when the service starts or manually call the command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import {verifyFromFile} from 'ip2region.js';
|
||||||
|
|
||||||
|
try {
|
||||||
|
verifyFromFile(dbPath);
|
||||||
|
} catch (e) {
|
||||||
|
// Applicability verification failed!!!
|
||||||
|
// The current query client implementation is not applicable for queries on the xdb file specified by dbPath.
|
||||||
|
// You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation that fits dbPath.
|
||||||
|
console.log(`binding is not applicable for xdb file '${dbPath}': ${e.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verification passed. The current Searcher can be safely used for query operations on the xdb pointed to by dbPath.
|
||||||
|
```
|
||||||
|
|
||||||
|
### File-Only Query
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import {newWithFileOnly} from 'ip2region.js';
|
||||||
|
|
||||||
|
// 1. Create a file-only query object using the version and dbPath mentioned above
|
||||||
|
let searcher;
|
||||||
|
try {
|
||||||
|
searcher = newWithFileOnly(version, dbPath);
|
||||||
|
} catch(e) {
|
||||||
|
console.log(`failed to newWithFileOnly: ${err.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// 2. Query; the interface is the same for both IPv4 and IPv6 addresses
|
||||||
|
let ip = "1.2.3.4";
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
try {
|
||||||
|
let region = await searcher.search(ip);
|
||||||
|
console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`);
|
||||||
|
} catch(e) {
|
||||||
|
console.log(`${err.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Close resources
|
||||||
|
searcher.close();
|
||||||
|
|
||||||
|
// Note: Each thread needs to create an independent Searcher object separately
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
We can pre-load the `VectorIndex` data from the `xdb` file and cache it globally. Using a global VectorIndex cache every time a Searcher object is created can reduce one fixed IO operation, thereby accelerating queries and reducing IO pressure.
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import {loadVectorIndexFromFile, newWithVectorIndex} from 'ip2region.js';
|
||||||
|
|
||||||
|
// 1. Pre-load VectorIndex cache from dbPath and keep this data as a global variable for subsequent repeated use.
|
||||||
|
let vIndex;
|
||||||
|
try {
|
||||||
|
vIndex = loadVectorIndexFromFile(dbPath);
|
||||||
|
} catch (e) {
|
||||||
|
console.log(`failed to load vector index from ${dbPath}: ${e.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Create a query object with VectorIndex cache using the global vIndex.
|
||||||
|
let searcher;
|
||||||
|
try {
|
||||||
|
searcher = newWithVectorIndex(version, dbPath, vIndex);
|
||||||
|
} catch(e) {
|
||||||
|
console.log(`failed to newWithVectorIndex: ${err.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// 3. Query; the interface is the same for both IPv4 and IPv6 addresses
|
||||||
|
let ip = "1.2.3.4";
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
try {
|
||||||
|
let region = await searcher.search(ip);
|
||||||
|
console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`);
|
||||||
|
} catch(e) {
|
||||||
|
console.log(`${err.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Close resources
|
||||||
|
searcher.close();
|
||||||
|
|
||||||
|
// Note: Each thread needs to create a separate independent Searcher object, but they all share the global read-only vIndex cache.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching the entire `xdb` file
|
||||||
|
|
||||||
|
We can also pre-load the data of the entire xdb file into memory and then create a query object based on this data to achieve a completely memory-based query, similar to the previous memory search.
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
import {loadContentFromFile, newWithBuffer} from 'ip2region.js';
|
||||||
|
|
||||||
|
// 1. Load the entire xdb from dbPath into memory.
|
||||||
|
let cBuffer;
|
||||||
|
try {
|
||||||
|
cBuffer = loadContentFromFile(dbPath);
|
||||||
|
} catch (e) {
|
||||||
|
console.log(`failed to load content from ${dbPath}: ${e.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Use the cBuff above to create a completely memory-based query object.
|
||||||
|
let searcher;
|
||||||
|
try {
|
||||||
|
searcher = newWithBuffer(version, cBuffer);
|
||||||
|
} catch(e) {
|
||||||
|
console.log(`failed to newWithBuffer: ${err.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Query; the interface is the same for both IPv4 and IPv6 addresses
|
||||||
|
let ip = "1.2.3.4";
|
||||||
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
try {
|
||||||
|
let region = await searcher.search(ip);
|
||||||
|
console.log(`search(${ip}): {region: ${region}`);
|
||||||
|
} catch(e) {
|
||||||
|
console.log(`${err.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Close resources - This searcher object is safe for concurrent use; close the searcher only when the entire service shuts down.
|
||||||
|
// searcher.close();
|
||||||
|
|
||||||
|
// Note: For concurrent use, the query object created with the entire xdb data cache can be safely used concurrently, meaning you can make this searcher object a global object for cross-thread access.
|
||||||
|
```
|
||||||
|
|
||||||
|
# Query Test
|
||||||
|
|
||||||
|
You can test queries using the `node tests/search.app.js` command:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ javascript git:(fr_javascript_ipv6) node tests/search.app.js
|
||||||
|
usage: Usage node tests/search.app.js [command options]
|
||||||
|
|
||||||
|
ip2region search script
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
--db DB ip2region binary xdb file path
|
||||||
|
--cache-policy CACHE_POLICY
|
||||||
|
cache policy: file/vectorIndex/content, default: vectorIndex
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: Using the default data/ip2region_v4.xdb file for IPv4 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ javascript git:(fr_javascript_ipv6) ✗ node tests/search.app.js --db=../../data/ip2region_v4.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 657.035 μs}
|
||||||
|
ip2region>> 113.118.113.77
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, ioCount: 2, took: 169.927 μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: Using the default data/ip2region_v6.xdb file for IPv6 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ javascript git:(fr_javascript_ipv6) ✗ node tests/search.app.js --db=../../data/ip2region_v6.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 98.953 μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, ioCount: 13, took: 287.703 μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
Enter an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the effects of the three different cache implementations.
|
||||||
|
|
||||||
|
# bench Test
|
||||||
|
|
||||||
|
You can perform a bench test via the `node tests/bench.app.js` command, which ensures the `xdb` file is error-free and evaluates query performance:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ javascript git:(fr_javascript_ipv6) ✗ node tests/bench.app.js
|
||||||
|
usage: Usage node tests/bench.app.js [command options]
|
||||||
|
|
||||||
|
ip2region bench script
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
--db DB ip2region binary xdb file path
|
||||||
|
--src SRC source ip text file path
|
||||||
|
--cache-policy CACHE_POLICY
|
||||||
|
cache policy: file/vectorIndex/content, default: vectorIndex
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: Perform an IPv4 bench test using the default data/ip2region_v4.xdb and data/ipv4_source.txt files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node tests/bench.app.js --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Example: Perform an IPv6 bench test using the default data/ip2region_v6.xdb and data/ipv6_source.txt files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node tests/bench.app.js --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
You can test the effects of the three different cache implementations by setting `cache-policy` to file/vectorIndex/content.
|
||||||
|
@Note: Ensure the src file used for bench is the same source file used to generate the corresponding xdb file.
|
||||||
|
|
||||||
|
### Third-party Library Support:
|
||||||
|
|
||||||
|
1. [ts-ip2region2](https://github.com/Steven-Qiang/ts-ip2region2) - Based on the official C extension, providing higher execution efficiency than pure JS.
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# ip2region xdb javascript 查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region javascript 查询客户端
|
||||||
|
|
||||||
# 使用方式
|
# 使用方式
|
||||||
|
|
||||||
|
|
@ -67,9 +69,9 @@ try {
|
||||||
|
|
||||||
// 2、查询,IPv4 或者 IPv6 的地址都是同一个接口
|
// 2、查询,IPv4 或者 IPv6 的地址都是同一个接口
|
||||||
let ip = "1.2.3.4";
|
let ip = "1.2.3.4";
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
try {
|
try {
|
||||||
let region = searcher.search(ip);
|
let region = await searcher.search(ip);
|
||||||
console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`);
|
console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`);
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
console.log(`${err.message}`);
|
console.log(`${err.message}`);
|
||||||
|
|
@ -99,7 +101,7 @@ try {
|
||||||
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
// 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。
|
||||||
let searcher;
|
let searcher;
|
||||||
try {
|
try {
|
||||||
searcher = newWithVectorIndex(version, vIndex, dbPath);
|
searcher = newWithVectorIndex(version, dbPath, vIndex);
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
console.log(`failed to newWithVectorIndex: ${err.message}`);
|
console.log(`failed to newWithVectorIndex: ${err.message}`);
|
||||||
return;
|
return;
|
||||||
|
|
@ -108,9 +110,9 @@ try {
|
||||||
|
|
||||||
// 3、查询,IPv4 或者 IPv6 的地址都是同一个接口
|
// 3、查询,IPv4 或者 IPv6 的地址都是同一个接口
|
||||||
let ip = "1.2.3.4";
|
let ip = "1.2.3.4";
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
try {
|
try {
|
||||||
let region = searcher.search(ip);
|
let region = await searcher.search(ip);
|
||||||
console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`);
|
console.log(`search(${ip}): {region: ${region}, ioCount: ${searcher.getIOCount()}}`);
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
console.log(`${err.message}`);
|
console.log(`${err.message}`);
|
||||||
|
|
@ -122,7 +124,7 @@ searcher.close();
|
||||||
// 备注:每个线程需要单独创建一个独立的 Searcher 对象,但是都共享全局的只读 vIndex 缓存。
|
// 备注:每个线程需要单独创建一个独立的 Searcher 对象,但是都共享全局的只读 vIndex 缓存。
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
我们也可以预先加载整个 xdb 文件的数据到内存,然后基于这个数据创建查询对象来实现完全基于内存的查询,类似之前的 memory search。
|
我们也可以预先加载整个 xdb 文件的数据到内存,然后基于这个数据创建查询对象来实现完全基于内存的查询,类似之前的 memory search。
|
||||||
```javascript
|
```javascript
|
||||||
|
|
@ -148,9 +150,9 @@ try {
|
||||||
|
|
||||||
// 3、查询,IPv4 或者 IPv6 的地址都是同一个接口
|
// 3、查询,IPv4 或者 IPv6 的地址都是同一个接口
|
||||||
let ip = "1.2.3.4";
|
let ip = "1.2.3.4";
|
||||||
// ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
try {
|
try {
|
||||||
let region = searcher.search(ip);
|
let region = await searcher.search(ip);
|
||||||
console.log(`search(${ip}): {region: ${region}`);
|
console.log(`search(${ip}): {region: ${region}`);
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
console.log(`${err.message}`);
|
console.log(`${err.message}`);
|
||||||
|
|
@ -186,7 +188,9 @@ ip2region xdb searcher test program
|
||||||
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
ip2region>> 1.2.3.4
|
ip2region>> 1.2.3.4
|
||||||
{region: 美国|华盛顿|0|谷歌, ioCount: 7, took: 0.430357 μs}
|
{region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 657.035 μs}
|
||||||
|
ip2region>> 113.118.113.77
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, ioCount: 2, took: 169.927 μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
例如:使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试:
|
例如:使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试:
|
||||||
|
|
@ -196,7 +200,9 @@ ip2region xdb searcher test program
|
||||||
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, ioCount: 14, took: 4.727663 μs}
|
{region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 98.953 μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, ioCount: 13, took: 287.703 μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
输入 ip 即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
|
输入 ip 即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "ip2region.js",
|
"name": "ip2region.js",
|
||||||
"version": "3.1.6",
|
"version": "3.1.8",
|
||||||
"description": "official javascript binding for ip2region with both IPv4 and IPv6 supported ",
|
"description": "official javascript binding for ip2region with both IPv4 and IPv6 supported ",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
|
|
@ -31,11 +31,20 @@
|
||||||
"ipv6-search"
|
"ipv6-search"
|
||||||
],
|
],
|
||||||
"author": "lionsoul2014",
|
"author": "lionsoul2014",
|
||||||
"license": "ISC",
|
"license": "Apache-2.0 OR MIT",
|
||||||
"bugs": {
|
"bugs": {
|
||||||
"url": "https://github.com/lionsoul2014/ip2region/issues"
|
"url": "https://github.com/lionsoul2014/ip2region/issues"
|
||||||
},
|
},
|
||||||
"homepage": "https://github.com/lionsoul2014/ip2region#readme",
|
"homepage": "https://github.com/lionsoul2014/ip2region#readme",
|
||||||
|
"files": [
|
||||||
|
"index.js",
|
||||||
|
"index.d.ts",
|
||||||
|
"searcher.js",
|
||||||
|
"util.js",
|
||||||
|
"README.md",
|
||||||
|
"README_zh.md",
|
||||||
|
"LICENSE.md"
|
||||||
|
],
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/jest": "^30.0.0",
|
"@types/jest": "^30.0.0",
|
||||||
"argparse": "^2.0.1",
|
"argparse": "^2.0.1",
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,12 @@ export class Searcher {
|
||||||
}
|
}
|
||||||
|
|
||||||
// console.log(`sPtr: ${sPtr}, ePtr: ${ePtr}`);
|
// console.log(`sPtr: ${sPtr}, ePtr: ${ePtr}`);
|
||||||
|
// @Note: ptr validate, zero ptr means source data missing
|
||||||
|
// so we could just stop here and return an empty string.
|
||||||
|
if (sPtr == 0 || ePtr == 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
// binary search the segment index block to get the region info
|
// binary search the segment index block to get the region info
|
||||||
const bytes = ipBytes.length, dBytes = ipBytes.length << 1;
|
const bytes = ipBytes.length, dBytes = ipBytes.length << 1;
|
||||||
const indexSize = this.version.indexSize;
|
const indexSize = this.version.indexSize;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,239 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region lua query client
|
||||||
|
|
||||||
|
#### Note: Please prioritize the use of the lua_c extension query client, as its performance is much faster than the pure lua implementation!!!
|
||||||
|
|
||||||
|
# Version Compatibility
|
||||||
|
|
||||||
|
This implementation is compatible with lua `5.3` and `5.4`, and no longer provides compatibility for lower versions. If you need to use it under lower versions of Lua, please consider using the `[lua_c](../lua_c/)` extension.
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### About Query API
|
||||||
|
|
||||||
|
The prototype of the query API is as follows:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- Query via IP string
|
||||||
|
search_by_string(ip_string) (region, error)
|
||||||
|
-- Query via bytes IP returned by parse_ip
|
||||||
|
search(ip_bytes) (region, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
If the query fails, it returns a non-`nil` error string. If successful, it returns the `region` information as a string. If the IP address cannot be found, it returns an empty string `""`.
|
||||||
|
|
||||||
|
### About IPv4 and IPv6
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- For IPv4: Set xdb path to the v4 xdb file, specify IP version as Version.IPv4
|
||||||
|
local dbPath = "../../data/ip2region_v4.xdb" -- or your ipv4 xdb path
|
||||||
|
local version = xdb.IPv4
|
||||||
|
|
||||||
|
-- For IPv6: Set xdb path to the v6 xdb file, specify IP version as Version.IPv6
|
||||||
|
local dbPath = "../../data/ip2region_v6.xdb" -- or your ipv6 xdb path
|
||||||
|
local version = xdb.IPv6
|
||||||
|
|
||||||
|
-- The IP version of the xdb specified by dbPath must match the version specified, otherwise an error will occur during query execution
|
||||||
|
-- Note: The following demonstration directly uses the dbPath and version variables
|
||||||
|
```
|
||||||
|
|
||||||
|
### File Verification
|
||||||
|
|
||||||
|
It is recommended that you actively verify the suitability of the xdb file, as some new features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You don't need to verify every time; for example, verify when the service starts or by manually calling the verification command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require('xdb_searcher')
|
||||||
|
|
||||||
|
local err = xdb.verify(dbPath);
|
||||||
|
if err ~= nil then
|
||||||
|
-- Suitability verification failed!!!
|
||||||
|
-- The current query client implementation is not suitable for the xdb file specified by dbPath.
|
||||||
|
-- You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with dbPath.
|
||||||
|
print(string.format("binding is not applicable for xdb file '%s': %s", dbPath, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- Verification passed, the current Searcher can safely be used for query operations on the xdb pointed to by dbPath
|
||||||
|
```
|
||||||
|
|
||||||
|
### Entirely File-Based Query
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- 1. Create an entirely file-based query object using the version and dbPath mentioned above
|
||||||
|
local searcher, err = xdb.new_with_file_only(version, db_path)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to create searcher: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 2. Call the query API; with both IPv4 and IPv6 addresses supported
|
||||||
|
local ip_str = "1.2.3.4"
|
||||||
|
-- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6
|
||||||
|
local s_time = xdb.now()
|
||||||
|
region, err = searcher:search_by_string(ip_str)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time))
|
||||||
|
|
||||||
|
-- 3. Close resources
|
||||||
|
searcher:close()
|
||||||
|
|
||||||
|
-- Note: For concurrent use, each coroutine needs to create a separate xdb query object
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
If supported by your `lua` environment, you can pre-load the vectorIndex cache and make it a global variable. Using the global vectorIndex every time a Searcher is created can reduce one fixed IO operation, thereby accelerating queries and reducing IO pressure.
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- 1. Load vectorIndex cache from the specified db_path and make the v_index object below a global variable.
|
||||||
|
-- vectorIndex only needs to be loaded once; it is recommended to load it as a global object when the service starts.
|
||||||
|
v_index, err = xdb.load_vector_index(dbPath)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to load vector index from '%s'", db_path))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 2. Use the global v_index to create a query object with vectorIndex cache.
|
||||||
|
searcher, err = xdb.new_with_vector_index(version, dbPath, v_index)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to create vector index searcher: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 3. Call the query API; the same interface is used for both IPv4 and IPv6
|
||||||
|
local ip_str = "1.2.3.4"
|
||||||
|
-- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6
|
||||||
|
local s_time = xdb.now()
|
||||||
|
region, err = searcher:search_by_string(ip_str)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time))
|
||||||
|
|
||||||
|
-- 4. Close resources
|
||||||
|
searcher:close()
|
||||||
|
|
||||||
|
-- Note: For concurrent use, each coroutine needs to create a separate xdb query object, but they share the global v_index object
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching the Entire `xdb` File
|
||||||
|
|
||||||
|
If supported by your `lua` environment, you can pre-load the entire xdb data into memory to achieve completely memory-based queries, similar to the previous memory search.
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- 1. Load the entire xdb into memory from the specified dbPath.
|
||||||
|
-- xdb content only needs to be loaded once; it is recommended to load it as a global object when the service starts.
|
||||||
|
content, err = xdb.load_content(dbPath)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to load xdb content: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 2. Use the global content to create an entirely memory-based query object.
|
||||||
|
searcher, err = xdb.new_with_buffer(version, content)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to create content buffer searcher: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 3. Call the query API; the same interface is used for both IPv4 and IPv6
|
||||||
|
local ip_str = "1.2.3.4"
|
||||||
|
-- local ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" -- IPv6
|
||||||
|
local s_time = xdb.now()
|
||||||
|
region, err = searcher:search_by_string(ip_str)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
print(string.format("{region: %s, io_count: %d, took: %.5f μs}", region, searcher:get_io_count(), xdb.now() - s_time))
|
||||||
|
|
||||||
|
-- 4. Close resources - This searcher object can be safely used for concurrency; close the searcher only when the entire service is shut down
|
||||||
|
-- searcher:close()
|
||||||
|
|
||||||
|
-- Note: For concurrent use, query objects created with the entire xdb cache can be safely used concurrently.
|
||||||
|
-- It is recommended to create a global searcher object when the service starts and then use it globally and concurrently.
|
||||||
|
```
|
||||||
|
|
||||||
|
# Query Testing
|
||||||
|
|
||||||
|
Perform query tests via the `lua search_test.lua` script:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua
|
||||||
|
lua search_test.lua [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: using the default data/ip2region_v4.xdb file for IPv4 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua --db=../../data/ip2region_v4.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 0μs}
|
||||||
|
ip2region>> 113.118.113.77
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 2, took: 0μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: using the default data/ip2region_v6.xdb file for IPv6 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua git:(fr_lua_ipv6) ✗ lua search_test.lua --db=../../data/ip2region_v6.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 0μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, io_count: 13, took: 0μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
Enter an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the efficiency of the three different cache implementations.
|
||||||
|
|
||||||
|
# Bench Testing
|
||||||
|
|
||||||
|
Perform automatic bench testing via the `lua bench_test.lua` script. This ensures that the `xdb` file has no errors and tests average query performance through a large number of queries:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua git:(fr_lua_ipv6) ✗ lua bench_test.lua
|
||||||
|
lua bench_test.lua [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--src string source ip text file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: perform IPv4 bench testing using default data/ip2region_v4.xdb and data/ipv4_source.txt files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
lua bench_test.lua --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: perform IPv6 bench testing using default data/ip2region_v6.xdb and data/ipv6_source.txt files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
lua bench_test.lua --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
You can test the performance of the three different cache implementations (file/vectorIndex/content) by setting the `cache-policy` parameter.
|
||||||
|
@Note: Please note that the src file used for the bench must be the same source file used to generate the corresponding xdb file.
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# ip2region xdb lua 查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region lua 查询客户端
|
||||||
|
|
||||||
#### 备注:请优先使用 lua_c 扩展 xdb 查询客户端,性能比纯 lua 实现的要快很多!!!
|
#### 备注:请优先使用 lua_c 扩展 xdb 查询客户端,性能比纯 lua 实现的要快很多!!!
|
||||||
|
|
||||||
|
|
@ -118,7 +120,7 @@ searcher:close()
|
||||||
-- 备注:并发使用,每个协程需要创建单独的 xdb 查询对象,但是共享全局的 v_index 对象
|
-- 备注:并发使用,每个协程需要创建单独的 xdb 查询对象,但是共享全局的 v_index 对象
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
如果你的 `lua` 母环境支持,可以预先加载整个 xdb 的数据到内存,这样可以实现完全基于内存的查询,类似之前的 memory search 查询。
|
如果你的 `lua` 母环境支持,可以预先加载整个 xdb 的数据到内存,这样可以实现完全基于内存的查询,类似之前的 memory search 查询。
|
||||||
```lua
|
```lua
|
||||||
|
|
@ -177,7 +179,9 @@ ip2region xdb searcher test program
|
||||||
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
ip2region>> 1.2.3.4
|
ip2region>> 1.2.3.4
|
||||||
{region: 美国|华盛顿|0|谷歌, io_count: 7, took: 0μs}
|
{region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 0μs}
|
||||||
|
ip2region>> 113.118.113.77
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 2, took: 0μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
例如:使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试:
|
例如:使用默认的 data/ip2region_v6.xdb 文件进行 IPv6 的查询测试:
|
||||||
|
|
@ -187,7 +191,9 @@ ip2region xdb searcher test program
|
||||||
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, io_count: 8, took: 0μs}
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 0μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, io_count: 13, took: 0μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
||||||
|
|
@ -129,6 +129,13 @@ function xdb:search(ip_bytes)
|
||||||
end
|
end
|
||||||
|
|
||||||
-- print(string.format("s_ptr: %d, e_ptr: %d", s_ptr, e_ptr))
|
-- print(string.format("s_ptr: %d, e_ptr: %d", s_ptr, e_ptr))
|
||||||
|
-- @Note: ptr validate, zero ptr means source data missing
|
||||||
|
-- so we could just stop here and return an empty string.
|
||||||
|
if s_ptr == 0 or e_ptr == 0 then
|
||||||
|
return "", nil
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
-- binary search to get the data
|
-- binary search to get the data
|
||||||
local index_size, ip_sub_compare = version.index_size, version.ip_sub_compare
|
local index_size, ip_sub_compare = version.index_size, version.ip_sub_compare
|
||||||
local bytes, d_bytes = version.bytes, version.bytes << 1
|
local bytes, d_bytes = version.bytes, version.bytes << 1
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,290 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region lua c extension query client
|
||||||
|
|
||||||
|
# Version Compatibility
|
||||||
|
|
||||||
|
This implementation is compatible with lua `5.1`, `5.2`, `5.3`, and `5.4`.
|
||||||
|
|
||||||
|
# Compilation and Installation
|
||||||
|
|
||||||
|
### Default Compilation
|
||||||
|
|
||||||
|
Use the following commands to compile and install the default `Lua5.4` version of the extension:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# cd to the root directory of lua_c binding
|
||||||
|
make
|
||||||
|
sudo make install
|
||||||
|
```
|
||||||
|
|
||||||
|
### Specify Lua Version
|
||||||
|
|
||||||
|
Specify the Lua version for compilation using the `LuaVersion` parameter, for example: `5.1` / `5.2` / `5.3` / `5.4`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# cd to the root directory of lua_c binding
|
||||||
|
# For example, compile the extension compatible with version 5.1
|
||||||
|
make LuaVersion=5.1
|
||||||
|
sudo make install
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: Please use the same version of `lua` to run the following tests as the one used to compile the extension. For example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Compile extension using lua 5.1
|
||||||
|
make LuaVersion=5.1
|
||||||
|
|
||||||
|
# Run query test using lua5.1
|
||||||
|
lua5.1 search_test.py --db=../../data/ip2region_v4.xdb
|
||||||
|
```
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### About Query API
|
||||||
|
|
||||||
|
The prototype of the query API is as follows:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- Query via IP string or binary IP parsed by xdb.parse_ip
|
||||||
|
search(ip_string | ip_bytes) (region, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
If the query fails, `error` will be a non-`nil` error description string. If successful, it returns the `region` information as a string. If the IP address is not found, it returns an empty string `""`.
|
||||||
|
|
||||||
|
### About IPv4 and IPv6
|
||||||
|
|
||||||
|
This xdb query client implementation supports both IPv4 and IPv6 queries. Usage is as follows:
|
||||||
|
|
||||||
|
```lua
|
||||||
|
-- Import xdb searcher extension
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- For IPv4: Set xdb path to v4 xdb file, specify IP version as IPv4
|
||||||
|
local db_path = "../../data/ip2region_v4.xdb" -- or your ipv4 xdb path
|
||||||
|
local version = xdb.IPv4
|
||||||
|
|
||||||
|
-- For IPv6: Set xdb path to v6 xdb file, specify IP version as IPv6
|
||||||
|
local db_path = "../../data/ip2region_v6.xdb"; -- or your ipv6 xdb path
|
||||||
|
local version = xdb.IPv6
|
||||||
|
|
||||||
|
-- The IP version of the xdb specified by db_path must match the version specified, otherwise an error will occur during query execution
|
||||||
|
-- Note: The following demonstration directly uses the db_path and version variables
|
||||||
|
```
|
||||||
|
|
||||||
|
### XDB File Verification
|
||||||
|
|
||||||
|
It is recommended to actively verify the suitability of the xdb file. New features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification helps avoid unpredictable errors during runtime. You don't need to verify every time; for example, verify when the service starts or by manually calling the verification command. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- verify the xdb
|
||||||
|
if xdb.verify(db_path) == false then
|
||||||
|
-- Suitability verification failed!!!
|
||||||
|
-- The current query client implementation is not suitable for the xdb file specified by db_path.
|
||||||
|
-- You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with db_path.
|
||||||
|
print(string.format("failed to verify the xdb file: %s", db_path))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- Verification passed, the current Searcher can safely be used for query operations on the xdb pointed to by db_path
|
||||||
|
```
|
||||||
|
|
||||||
|
### Entirely File-Based Query
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- 1. Create a file-based xdb query object from db_path using version
|
||||||
|
local searcher, err = xdb.new_with_file_only(version, db_path)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to create searcher: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 2. Call the query API; both IPv4 and IPv6 are supported
|
||||||
|
local ip_str = "1.2.3.4"
|
||||||
|
-- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
|
local s_time = xdb.now()
|
||||||
|
region, err = searcher:search(ip_str)
|
||||||
|
local c_time = xdb.now() - s_time
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
print(string.format("{region: %s, took: %.5f μs}", region, c_time))
|
||||||
|
|
||||||
|
-- Note: For concurrent use, each coroutine needs to create a separate xdb query object
|
||||||
|
|
||||||
|
-- 3. Close the xdb searcher
|
||||||
|
searcher:close()
|
||||||
|
|
||||||
|
--
|
||||||
|
-- 4. Module resource cleanup, only call before the entire service is completely shut down
|
||||||
|
xdb.cleanup()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
If supported by your `lua` environment, you can pre-load the `vectorIndex` cache and make it a global variable. Using the global `vectorIndex` every time a Searcher is created can reduce one fixed IO operation, thereby accelerating queries and reducing IO pressure.
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- 1. Load VectorIndex cache from the specified db_path and make the v_index object below a global variable.
|
||||||
|
-- vectorIndex only needs to be loaded once; it is recommended to load it as a global object when the service starts.
|
||||||
|
v_index, err = xdb.load_vector_index(db_path)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to load vector index from '%s'", db_path))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 2. Use the global v_index to create a query object with VectorIndex cache.
|
||||||
|
searcher, err = xdb.new_with_vector_index(version, db_path, v_index)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to create vector index searcher: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 3. Call the query API; both IPv4 and IPv6 are supported
|
||||||
|
local ip_str = "1.2.3.4"
|
||||||
|
-- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
|
local s_time = xdb.now()
|
||||||
|
region, err = searcher:search(ip_str)
|
||||||
|
local c_time = xdb.now() = s_time
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
print(string.format("{region: %s, took: %.5f μs}", region, c_time))
|
||||||
|
|
||||||
|
-- Note: For concurrent use, each coroutine needs to create a separate xdb query object, but they share the global v_index object
|
||||||
|
|
||||||
|
-- 4. Close the xdb searcher
|
||||||
|
searcher:close()
|
||||||
|
|
||||||
|
--
|
||||||
|
-- 5. Module resource cleanup, only call before the entire service is completely shut down
|
||||||
|
xdb.cleanup()
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching the Entire `xdb` File
|
||||||
|
|
||||||
|
If supported by your `lua` environment, you can pre-load the entire xdb data into memory to achieve completely memory-based queries, similar to the previous memory search.
|
||||||
|
|
||||||
|
```lua
|
||||||
|
local xdb = require("xdb_searcher")
|
||||||
|
|
||||||
|
-- 1. Load the entire xdb into memory from the specified db_path.
|
||||||
|
-- xdb content only needs to be loaded once; it is recommended to load it as a global object when the service starts.
|
||||||
|
local content = xdb.load_content(db_path)
|
||||||
|
if content == nil then
|
||||||
|
print(string.format("failed to load xdb content from '%s'", db_path))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 2. Use the global content to create a query object based entirely on memory.
|
||||||
|
searcher, err = xdb.new_with_buffer(version, content)
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to create content buffer searcher: %s", err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
-- 3. Call the query API; both IPv4 and IPv6 are supported
|
||||||
|
local ip_str = "1.2.3.4"
|
||||||
|
-- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
|
local s_time = xdb.now()
|
||||||
|
region, err = searcher:search(ip_str)
|
||||||
|
local c_time = xdb.now() - s_time
|
||||||
|
if err ~= nil then
|
||||||
|
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
print(string.format("{region: %s, took: %.5f μs}", region, c_time))
|
||||||
|
|
||||||
|
-- Note: For concurrent use, query objects created with the entire xdb cache can be safely used concurrently.
|
||||||
|
-- It is recommended to create a global searcher object when the service starts and then use it globally and concurrently.
|
||||||
|
|
||||||
|
-- 4. Close the xdb searcher
|
||||||
|
searcher:close()
|
||||||
|
|
||||||
|
--
|
||||||
|
-- 5. Module resource cleanup, only call before the entire service is completely shut down
|
||||||
|
xdb.cleanup()
|
||||||
|
```
|
||||||
|
|
||||||
|
# Query Testing
|
||||||
|
|
||||||
|
Perform query tests via the `search_test.lua` script:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./search_test.lua
|
||||||
|
lua search_test.lua [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: using the default `data/ip2region_v4.xdb` for IPv4 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./search_test.lua --db=../../data/ip2region_v4.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 17μs}
|
||||||
|
ip2region>> 120.229.45.2
|
||||||
|
{region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 40μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: using the default `data/ip2region_v6.xdb` for IPv6 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua_c git:(master) lua ./search_test.lua --db=../../data/ip2region_v6.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> ::
|
||||||
|
{region: , io_count: 1, took: 48μs}
|
||||||
|
ip2region>> 240e:3b7:3276:33b0:958f:f34c:d04f:f6a
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 52μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, io_count: 13, took: 35μs}
|
||||||
|
```
|
||||||
|
|
||||||
|
Enter an IP to perform a query test. You can also set `cache-policy` to `file`/`vectorIndex`/`content` respectively to test the efficiency of the three different cache implementations.
|
||||||
|
|
||||||
|
# Bench Testing
|
||||||
|
|
||||||
|
Perform automatic bench testing via the `bench_test.lua` script. This ensures that the `xdb` file has no errors and tests average query performance through a large number of queries:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua
|
||||||
|
lua bench_test.lua [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--src string source ip text file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: perform IPv4 bench testing using default `data/ip2region_v4.xdb` and `data/ipv4_source.txt`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
Bench finished, {cachePolicy: vectorIndex, total: 1367686, took: 8.593 s, cost: 5.433 μs/op}
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: perform IPv6 bench testing using default `data/ip2region_v6.xdb` and `data/ipv6_source.txt`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ lua_c git:(fr_lua_c_ipv6) ✗ lua ./bench_test.lua --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
Bench finished, {cachePolicy: vectorIndex, total: 34159862, took: 829.008 s, cost: 23.176 μs/op}
|
||||||
|
```
|
||||||
|
|
||||||
|
You can test the performance of the three different cache implementations (`file`/`vectorIndex`/`content`) by setting the `cache-policy` parameter.
|
||||||
|
@Note: Please ensure that the `src` file used for the bench is the same source file used to generate the corresponding `xdb` file.
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# ip2region xdb lua c 扩展查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region lua c 扩展查询客户端
|
||||||
|
|
||||||
# 版本兼容
|
# 版本兼容
|
||||||
该实现兼容 lua `5.1`,`5.2`,`5.3`, `5.4`
|
该实现兼容 lua `5.1`,`5.2`,`5.3`, `5.4`
|
||||||
|
|
@ -91,7 +93,7 @@ end
|
||||||
|
|
||||||
-- 2、调用查询 API 进行查询,IPv4 和 IPv6 都支持
|
-- 2、调用查询 API 进行查询,IPv4 和 IPv6 都支持
|
||||||
local ip_str = "1.2.3.4"
|
local ip_str = "1.2.3.4"
|
||||||
-- ip_str = "2001:4:112:ffff:ffff:ffff:ffff:ffff" // IPv6
|
-- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
local s_time = xdb.now()
|
local s_time = xdb.now()
|
||||||
region, err = searcher:search(ip_str)
|
region, err = searcher:search(ip_str)
|
||||||
local c_time = xdb.now() - s_time
|
local c_time = xdb.now() - s_time
|
||||||
|
|
@ -135,7 +137,7 @@ end
|
||||||
|
|
||||||
-- 3、调用查询 API ,IPv4 和 IPv6 都支持
|
-- 3、调用查询 API ,IPv4 和 IPv6 都支持
|
||||||
local ip_str = "1.2.3.4"
|
local ip_str = "1.2.3.4"
|
||||||
-- ip_str = "2001:4:112:ffff:ffff:ffff:ffff:ffff" // IPv6
|
-- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
local s_time = xdb.now()
|
local s_time = xdb.now()
|
||||||
region, err = searcher:search(ip_str)
|
region, err = searcher:search(ip_str)
|
||||||
local c_time = xdb.now() = s_time
|
local c_time = xdb.now() = s_time
|
||||||
|
|
@ -156,7 +158,7 @@ searcher:close()
|
||||||
xdb.cleanup()
|
xdb.cleanup()
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
如果你的 `lua` 母环境支持,可以预先加载整个 xdb 的数据到内存,这样可以实现完全基于内存的查询,类似之前的 memory search 查询。
|
如果你的 `lua` 母环境支持,可以预先加载整个 xdb 的数据到内存,这样可以实现完全基于内存的查询,类似之前的 memory search 查询。
|
||||||
```lua
|
```lua
|
||||||
|
|
@ -179,7 +181,7 @@ end
|
||||||
|
|
||||||
-- 3、调用查询 API ,IPv4 和 IPv6 都支持
|
-- 3、调用查询 API ,IPv4 和 IPv6 都支持
|
||||||
local ip_str = "1.2.3.4"
|
local ip_str = "1.2.3.4"
|
||||||
-- ip_str = "2001:4:112:ffff:ffff:ffff:ffff:ffff" // IPv6
|
-- ip_str = "240e:3b7:3272:d8d0:db09:c067:8d59:539e" // IPv6
|
||||||
local s_time = xdb.now()
|
local s_time = xdb.now()
|
||||||
region, err = searcher:search(ip_str)
|
region, err = searcher:search(ip_str)
|
||||||
local c_time = xdb.now() - s_time
|
local c_time = xdb.now() - s_time
|
||||||
|
|
@ -219,8 +221,10 @@ options:
|
||||||
ip2region xdb searcher test program
|
ip2region xdb searcher test program
|
||||||
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
source xdb: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, io_count: 5, took: 17μs}
|
||||||
ip2region>> 120.229.45.2
|
ip2region>> 120.229.45.2
|
||||||
{region: 中国|广东省|深圳市|移动, io_count: 3, took: 34μs}
|
{region: 中国|广东省|深圳市|移动|CN, io_count: 3, took: 40μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
例如:使用默认的 data/ip2region_v6.xdb 进行 IPv6 查询测试:
|
例如:使用默认的 data/ip2region_v6.xdb 进行 IPv6 查询测试:
|
||||||
|
|
@ -232,7 +236,9 @@ type 'quit' to exit
|
||||||
ip2region>> ::
|
ip2region>> ::
|
||||||
{region: , io_count: 1, took: 48μs}
|
{region: , io_count: 1, took: 48μs}
|
||||||
ip2region>> 240e:3b7:3276:33b0:958f:f34c:d04f:f6a
|
ip2region>> 240e:3b7:3276:33b0:958f:f34c:d04f:f6a
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, io_count: 8, took: 51μs}
|
{region: 中国|广东省|深圳市|电信|CN, io_count: 8, took: 52μs}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, io_count: 13, took: 35μs}
|
||||||
```
|
```
|
||||||
|
|
||||||
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
ARG NGINX_VERSION=1.29.6
|
||||||
|
FROM nginx:${NGINX_VERSION} AS build
|
||||||
|
ARG NGINX_VERSION
|
||||||
|
|
||||||
|
# prepare the build environment
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y build-essential libpcre2-dev zlib1g-dev libssl-dev git
|
||||||
|
|
||||||
|
WORKDIR /usr/src
|
||||||
|
RUN curl -LO https://github.com/nginx/nginx/releases/download/release-${NGINX_VERSION}/nginx-${NGINX_VERSION}.tar.gz && \
|
||||||
|
tar -zxf nginx-$NGINX_VERSION.tar.gz
|
||||||
|
|
||||||
|
COPY . /usr/src/ip2region
|
||||||
|
|
||||||
|
WORKDIR /usr/src/ip2region/binding/c
|
||||||
|
RUN make xdb_searcher_lib
|
||||||
|
|
||||||
|
# parameters for building dynamic modules
|
||||||
|
WORKDIR /usr/src
|
||||||
|
RUN nginx -V 2>&1 | grep 'configure arguments' | sed 's/ --/ \\\n --/g' | sed "s/pie'/pie' \\\/g" | grep -v 'configure arguments' >> /tmp/conf_arg
|
||||||
|
|
||||||
|
RUN echo \
|
||||||
|
' --add-dynamic-module=$(pwd)/../ip2region/binding/nginx \\\n' \
|
||||||
|
' --with-cc-opt="-I $(pwd)/../ip2region/binding/c/build/include" \\\n' \
|
||||||
|
' --with-ld-opt="-L $(pwd)/../ip2region/binding/c/build/lib"' >> /tmp/conf_arg
|
||||||
|
RUN cat /tmp/conf_arg
|
||||||
|
|
||||||
|
WORKDIR /usr/src/nginx-$NGINX_VERSION
|
||||||
|
RUN eval "./configure $(cat /tmp/conf_arg)"
|
||||||
|
|
||||||
|
RUN make modules && \
|
||||||
|
cp objs/ngx_http_ip2region_module.so /etc/nginx/modules
|
||||||
|
|
||||||
|
# for buildx export
|
||||||
|
FROM scratch AS export_so
|
||||||
|
ARG NGINX_VERSION
|
||||||
|
COPY --from=build /etc/nginx/modules/ngx_http_ip2region_module.so /ngx_http_ip2region_module.so
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
# nginx-ip2region
|
# nginx-ip2region
|
||||||
|
|
||||||
## build
|
## build
|
||||||
|
|
@ -5,16 +7,16 @@
|
||||||
```shell
|
```shell
|
||||||
$ mkdir -p workspace
|
$ mkdir -p workspace
|
||||||
$ cd workspace
|
$ cd workspace
|
||||||
$ wget https://nginx.org/download/nginx-1.23.4.tar.gz
|
$ wget https://nginx.org/download/nginx-1.23.6.tar.gz
|
||||||
$ tar -zxf nginx-1.23.4.tar.gz && rm -rf nginx-1.23.4.tar.gz
|
$ tar -zxf nginx-1.23.6.tar.gz && rm -rf nginx-1.23.6.tar.gz
|
||||||
$ git clone https://github.com/lionsoul2014/ip2region.git
|
$ git clone https://github.com/lionsoul2014/ip2region.git
|
||||||
$ cd ip2region/binding/c
|
$ cd ip2region/binding/c
|
||||||
$ make xdb_searcher_lib
|
$ make xdb_searcher_lib
|
||||||
$ cd ../../../nginx-1.23.4
|
$ cd ../../../nginx-1.23.6
|
||||||
$ ./configure \
|
$ ./configure \
|
||||||
--add-module=$(PWD)/../ip2region/binding/nginx \
|
--add-module=$(PWD)/../ip2region/binding/nginx \
|
||||||
--with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \
|
--with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \
|
||||||
--with-ld-opt="-L$(PWD)/../ip2region/binding/c/build/lib"
|
--with-ld-opt="-L $(PWD)/../ip2region/binding/c/build/lib"
|
||||||
$ make
|
$ make
|
||||||
$ make install
|
$ make install
|
||||||
```
|
```
|
||||||
|
|
@ -59,14 +61,39 @@ http {
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Copy `ip2region.xdb` to `nginx/config` folder, then restart nginx, the `region` data stored in `ip2region` variable
|
Copy `ip2region_v4.xdb` to `nginx/config` folder (rename name it to ip2region.xdb), then restart nginx, the `region` data stored in `ip2region` variable
|
||||||
|
|
||||||
nginx access log sample
|
nginx access log sample
|
||||||
|
|
||||||
```log
|
```log
|
||||||
{"remote_addr": "127.0.0.1", "region": "0|0|0|内网IP|内网IP", "http_x_forwarded_for": ""}
|
{"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""}
|
||||||
{"remote_addr": "127.0.0.1", "region": "0|0|0|内网IP|内网IP", "http_x_forwarded_for": ""}
|
{"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""}
|
||||||
|
```
|
||||||
|
|
||||||
|
Additionally, you can build the nginx dynamic module using the Dockerfile in the current directory.
|
||||||
|
|
||||||
|
> * The [buildx](https://github.com/docker/buildx) plugin is required to enable export functionality.
|
||||||
|
|
||||||
|
```shell
|
||||||
|
|
||||||
|
docker build -t export_so -o type=tar,dest=./so.tar .
|
||||||
|
# The final result is a dynamic module named ngx_http_ip2region_module.so.
|
||||||
|
tar xf so.tar && rm so.tar
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Made with ♥ by Wu Jian Ping
|
usage of dynamic modules
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
# nginx.conf
|
||||||
|
load_module /etc/nginx/my-modules/ngx_http_ip2region_module.so;
|
||||||
|
|
||||||
|
http {
|
||||||
|
# ...
|
||||||
|
ip2region_db /etc/nginx/conf.d/ip2region_v4.xdb content;
|
||||||
|
ip2region_db6 /etc/nginx/conf.d/ip2region_v6.xdb content;
|
||||||
|
# ...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,99 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# nginx-ip2region
|
||||||
|
|
||||||
|
## build
|
||||||
|
|
||||||
|
```shell
|
||||||
|
$ mkdir -p workspace
|
||||||
|
$ cd workspace
|
||||||
|
$ wget https://nginx.org/download/nginx-1.23.6.tar.gz
|
||||||
|
$ tar -zxf nginx-1.23.6.tar.gz && rm -rf nginx-1.23.6.tar.gz
|
||||||
|
$ git clone https://github.com/lionsoul2014/ip2region.git
|
||||||
|
$ cd ip2region/binding/c
|
||||||
|
$ make xdb_searcher_lib
|
||||||
|
$ cd ../../../nginx-1.23.6
|
||||||
|
$ ./configure \
|
||||||
|
--add-module=$(PWD)/../ip2region/binding/nginx \
|
||||||
|
--with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \
|
||||||
|
--with-ld-opt="-L$(PWD)/../ip2region/binding/c/build/lib"
|
||||||
|
$ make
|
||||||
|
$ make install
|
||||||
|
```
|
||||||
|
|
||||||
|
## nginx conf
|
||||||
|
|
||||||
|
> Syntax: `ip2region_db xdb_file_path [cache_policy Optional]`;
|
||||||
|
> Context: http
|
||||||
|
|
||||||
|
cache_policy: `file` or `vectorIndex` or `content`, default: `content`
|
||||||
|
|
||||||
|
Edit `nginx.conf` add `ip2region_db` directive
|
||||||
|
|
||||||
|
```nginx
|
||||||
|
...
|
||||||
|
http {
|
||||||
|
|
||||||
|
log_format main escape=json '{'
|
||||||
|
'"remote_addr": "$remote_addr", '
|
||||||
|
'"region": "$ip2region", '
|
||||||
|
'"http_x_forwarded_for": "$http_x_forwarded_for"'
|
||||||
|
'}';
|
||||||
|
|
||||||
|
access_log logs/access.log main;
|
||||||
|
|
||||||
|
# set xdb file path
|
||||||
|
ip2region_db ip2region.xdb;
|
||||||
|
# ip2region_db ip2region.xdb vectorIndex;
|
||||||
|
# ip2region_db ip2region.xdb file;
|
||||||
|
# ip2region_db ip2region.xdb content;
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
root html;
|
||||||
|
index index.html index.htm;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
Copy `ip2region_v4.xdb` to `nginx/config` folder (rename name it to ip2region.xdb), then restart nginx, the `region` data stored in `ip2region` variable
|
||||||
|
|
||||||
|
nginx access log sample
|
||||||
|
|
||||||
|
```log
|
||||||
|
{"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""}
|
||||||
|
{"remote_addr": "127.0.0.1", "region": "Reserved|Reserved|Reserved|0|0", "http_x_forwarded_for": ""}
|
||||||
|
```
|
||||||
|
|
||||||
|
另外,也可以使用当前路径的 Dockerfile 构建 nginx 动态模块
|
||||||
|
|
||||||
|
> *需要安装 [buildx](https://github.com/docker/buildx) 插件以支持导出*
|
||||||
|
|
||||||
|
```shell
|
||||||
|
|
||||||
|
docker build -t export_so -o type=tar,dest=./so.tar .
|
||||||
|
# 最终得到一个叫 ngx_http_ip2region_module.so 的动态模块
|
||||||
|
tar xf so.tar && rm so.tar
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
动态模块使用方式
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
# nginx.conf
|
||||||
|
load_module /etc/nginx/my-modules/ngx_http_ip2region_module.so;
|
||||||
|
|
||||||
|
http {
|
||||||
|
# ...
|
||||||
|
ip2region_db /etc/nginx/conf.d/ip2region_v4.xdb content;
|
||||||
|
ip2region_db6 /etc/nginx/conf.d/ip2region_v6.xdb content;
|
||||||
|
# ...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
@ -12,7 +12,7 @@ if test -n "$ngx_module_link"; then
|
||||||
ngx_module_name=$ngx_addon_name
|
ngx_module_name=$ngx_addon_name
|
||||||
ngx_module_deps="$NGX_HTTP_IP2REGION_DEPS"
|
ngx_module_deps="$NGX_HTTP_IP2REGION_DEPS"
|
||||||
ngx_module_srcs="$NGX_HTTP_IP2REGION_SRCS"
|
ngx_module_srcs="$NGX_HTTP_IP2REGION_SRCS"
|
||||||
ngx_module_libs="-lxdb_searcher"
|
ngx_module_libs="-lxdb"
|
||||||
|
|
||||||
. auto/module
|
. auto/module
|
||||||
else
|
else
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,9 @@
|
||||||
|
|
||||||
#include "ngx_http_ip2region_module.h"
|
#include "ngx_http_ip2region_module.h"
|
||||||
|
|
||||||
static ngx_int_t ngx_http_ua_parser_test_full_name(char *name);
|
static ngx_int_t ngx_http_ip2region_is_absolute_path(char *name);
|
||||||
|
static char *ngx_http_ip2region_init_searcher(ngx_conf_t *cf, char *db_name,
|
||||||
|
char *cache_policy, xdb_version_t *expected_version, const char *directive_name);
|
||||||
|
|
||||||
static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf);
|
static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf);
|
||||||
|
|
||||||
|
|
@ -28,6 +30,9 @@ static ngx_http_module_t ngx_http_ip2region_ctx = {
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
static char *ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
|
||||||
|
static char *ngx_http_ip2region_init_v6(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
|
||||||
|
|
||||||
static ngx_command_t ngx_http_ip2region_commands[] = {
|
static ngx_command_t ngx_http_ip2region_commands[] = {
|
||||||
{ ngx_string("ip2region_db"),
|
{ ngx_string("ip2region_db"),
|
||||||
NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12,
|
NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12,
|
||||||
|
|
@ -36,6 +41,13 @@ static ngx_command_t ngx_http_ip2region_commands[] = {
|
||||||
0,
|
0,
|
||||||
NULL },
|
NULL },
|
||||||
|
|
||||||
|
{ ngx_string("ip2region_db6"),
|
||||||
|
NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12,
|
||||||
|
ngx_http_ip2region_init_v6,
|
||||||
|
NGX_HTTP_MAIN_CONF_OFFSET,
|
||||||
|
0,
|
||||||
|
NULL },
|
||||||
|
|
||||||
ngx_null_command
|
ngx_null_command
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -66,25 +78,168 @@ static ngx_http_variable_t ngx_http_ip2region_vars[] = {
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
// 用于初始化带版本校验的搜索器的辅助函数
|
||||||
|
static char *
|
||||||
|
ngx_http_ip2region_init_searcher(ngx_conf_t *cf, char *db_name,
|
||||||
|
char *cache_policy, xdb_version_t *expected_version, const char *directive_name)
|
||||||
|
{
|
||||||
|
ip2region_searcher_t *ip2region_searcher;
|
||||||
|
int err;
|
||||||
|
char *db_path;
|
||||||
|
size_t len;
|
||||||
|
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) == NGX_OK) {
|
||||||
|
db_path = db_name;
|
||||||
|
} else { // relative path to conf directory
|
||||||
|
len = ngx_cycle->conf_prefix.len + strlen(db_name) + 1;
|
||||||
|
db_path = malloc(len);
|
||||||
|
if (db_path == NULL) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to allocate memory for db_path");
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
memset(db_path, '\0', len);
|
||||||
|
memcpy(db_path, ngx_cycle->conf_prefix.data, ngx_cycle->conf_prefix.len);
|
||||||
|
strcat(db_path, db_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
ip2region_searcher = ngx_palloc(cf->pool, sizeof(ip2region_searcher_t));
|
||||||
|
|
||||||
|
if(ip2region_searcher == NULL) {
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
ip2region_searcher->v_index = NULL;
|
||||||
|
ip2region_searcher->c_buffer = NULL;
|
||||||
|
|
||||||
|
// 检查XDB文件的版本信息以确定IP类型
|
||||||
|
xdb_header_t *header = xdb_load_header_from_file(db_path);
|
||||||
|
if (header == NULL) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to load xdb header from: %s", db_path);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
xdb_version_t *xdb_version = xdb_version_from_header(header);
|
||||||
|
if (xdb_version == NULL) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to determine xdb version from header: %s", db_path);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 验证XDB文件版本是否匹配
|
||||||
|
if (xdb_version->id != expected_version->id) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"%s expects %s xdb file, but got %s: %s",
|
||||||
|
directive_name, expected_version->name, xdb_version->name, db_path);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (strcmp(cache_policy, "file") == 0) {
|
||||||
|
err = xdb_new_with_file_only(xdb_version, &ip2region_searcher->searcher, db_path);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if (err != 0) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to create searcher: %s", db_path);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
} else if (strcmp(cache_policy, "vectorIndex") == 0) {
|
||||||
|
ip2region_searcher->v_index = xdb_load_vector_index_from_file(db_path);
|
||||||
|
if (ip2region_searcher->v_index == NULL) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to load vector index from: %s", db_path);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
err = xdb_new_with_vector_index(xdb_version, &ip2region_searcher->searcher, db_path, ip2region_searcher->v_index);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if (err != 0) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to create vector index cached searcher: %s", db_path);
|
||||||
|
xdb_free_vector_index((void *)ip2region_searcher->v_index);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
} else if (strcmp(cache_policy, "content") == 0) {
|
||||||
|
ip2region_searcher->c_buffer = xdb_load_content_from_file(db_path);
|
||||||
|
if (ip2region_searcher->c_buffer == NULL) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to load xdb content: %s", db_path);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
err = xdb_new_with_buffer(xdb_version, &ip2region_searcher->searcher, ip2region_searcher->c_buffer);
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if (err != 0) {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"failed to create content cached searcher: %s", db_path);
|
||||||
|
xdb_free_content((void *)ip2region_searcher->c_buffer);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
||||||
|
"invalid cache policy, options: file/vectorIndex/content");
|
||||||
|
xdb_free_header((void *)header);
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
return NGX_CONF_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(ngx_http_ip2region_is_absolute_path(db_name) != NGX_OK) {
|
||||||
|
free(db_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (char *)ip2region_searcher;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static char *
|
static char *
|
||||||
ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd,
|
ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd,
|
||||||
void *conf)
|
void *conf)
|
||||||
{
|
{
|
||||||
ngx_http_ip2region_conf_t *ip2region_cf;
|
ngx_http_ip2region_conf_t *ip2region_cf;
|
||||||
ip2region_searcher_t *ip2region_searcher;
|
|
||||||
char *db_name, *cache_policy;
|
char *db_name, *cache_policy;
|
||||||
ngx_str_t *value;
|
ngx_str_t *value;
|
||||||
int err;
|
char *result;
|
||||||
char *db_path;
|
|
||||||
size_t len;
|
|
||||||
ip2region_cf = conf;
|
ip2region_cf = conf;
|
||||||
|
|
||||||
if (ip2region_cf->ip2region_searcher) {
|
if (ip2region_cf->v4_searcher) {
|
||||||
return "is duplicate";
|
return "ip2region_db is duplicate";
|
||||||
}
|
}
|
||||||
|
|
||||||
value = cf->args->elts;
|
value = cf->args->elts;
|
||||||
|
|
||||||
db_name = (char *)value[1].data;
|
db_name = (char *)value[1].data;
|
||||||
|
|
||||||
// default cache_policy: content
|
// default cache_policy: content
|
||||||
|
|
@ -94,71 +249,49 @@ ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd,
|
||||||
cache_policy = (char *)value[2].data;
|
cache_policy = (char *)value[2].data;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(ngx_http_ua_parser_test_full_name(db_name) == NGX_OK) {
|
result = ngx_http_ip2region_init_searcher(cf, db_name, cache_policy,
|
||||||
db_path = db_name;
|
xdb_version_v4(), "ip2region_db");
|
||||||
} else { // relative path to conf directory
|
if (result == NGX_CONF_ERROR) {
|
||||||
len = ngx_cycle->conf_prefix.len + strlen(db_name) + 1;
|
|
||||||
db_path = malloc(len);
|
|
||||||
memset(db_path, '\0', len);
|
|
||||||
memcpy(db_path, ngx_cycle->conf_prefix.data, ngx_cycle->conf_prefix.len);
|
|
||||||
strcat(db_path, db_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
ip2region_searcher = ngx_palloc(cf->pool, sizeof(ip2region_searcher_t));
|
|
||||||
|
|
||||||
if(ip2region_searcher == NULL) {
|
|
||||||
return NGX_CONF_ERROR;
|
return NGX_CONF_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
ip2region_searcher->v_index = NULL;
|
ip2region_cf->v4_searcher = (ip2region_searcher_t *)result;
|
||||||
ip2region_searcher->c_buffer = NULL;
|
|
||||||
|
|
||||||
if (strcmp(cache_policy, "file") == 0) {
|
return NGX_CONF_OK;
|
||||||
err = xdb_new_with_file_only(&ip2region_searcher->searcher, db_path);
|
}
|
||||||
if (err != 0) {
|
|
||||||
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
|
||||||
"failed to create searcher: %s", db_path);
|
|
||||||
return NGX_CONF_ERROR;
|
|
||||||
}
|
|
||||||
} else if (strcmp(cache_policy, "vectorIndex") == 0) {
|
|
||||||
ip2region_searcher->v_index = xdb_load_vector_index_from_file(db_path);
|
|
||||||
if (ip2region_searcher->v_index == NULL) {
|
|
||||||
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
|
||||||
"failed to load vector index from: %s", db_path);
|
|
||||||
return NGX_CONF_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
err = xdb_new_with_vector_index(&ip2region_searcher->searcher, db_path, ip2region_searcher->v_index);
|
static char *
|
||||||
if (err != 0) {
|
ngx_http_ip2region_init_v6(ngx_conf_t *cf, ngx_command_t *cmd,
|
||||||
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
void *conf)
|
||||||
"failed to create vector index cached searcher: %s", db_path);
|
{
|
||||||
return NGX_CONF_ERROR;
|
ngx_http_ip2region_conf_t *ip2region_cf;
|
||||||
}
|
char *db_name, *cache_policy;
|
||||||
} else if (strcmp(cache_policy, "content") == 0) {
|
ngx_str_t *value;
|
||||||
ip2region_searcher->c_buffer = xdb_load_content_from_file(db_path);
|
char *result;
|
||||||
if (ip2region_searcher->c_buffer == NULL) {
|
|
||||||
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
|
||||||
"failed to load xdb content: %s", db_path);
|
|
||||||
return NGX_CONF_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
err = xdb_new_with_buffer(&ip2region_searcher->searcher, ip2region_searcher->c_buffer);
|
ip2region_cf = conf;
|
||||||
if (err != 0) {
|
|
||||||
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
if (ip2region_cf->v6_searcher) {
|
||||||
"failed to create content cached searcher: %s", db_path);
|
return "ip2region_db6 is duplicate";
|
||||||
return NGX_CONF_ERROR;
|
}
|
||||||
}
|
|
||||||
|
value = cf->args->elts;
|
||||||
|
db_name = (char *)value[1].data;
|
||||||
|
|
||||||
|
// default cache_policy: content
|
||||||
|
if(cf->args->nelts == 2) {
|
||||||
|
cache_policy = "content";
|
||||||
} else {
|
} else {
|
||||||
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
|
cache_policy = (char *)value[2].data;
|
||||||
"invalid cache policy `%V`, options: file/vectorIndex/content", &value[2]);
|
}
|
||||||
|
|
||||||
|
result = ngx_http_ip2region_init_searcher(cf, db_name, cache_policy,
|
||||||
|
xdb_version_v6(), "ip2region_db6");
|
||||||
|
if (result == NGX_CONF_ERROR) {
|
||||||
return NGX_CONF_ERROR;
|
return NGX_CONF_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
ip2region_cf->ip2region_searcher = ip2region_searcher;
|
ip2region_cf->v6_searcher = (ip2region_searcher_t *)result;
|
||||||
|
|
||||||
if(ngx_http_ua_parser_test_full_name(db_name) != NGX_OK) {
|
|
||||||
free(db_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
return NGX_CONF_OK;
|
return NGX_CONF_OK;
|
||||||
}
|
}
|
||||||
|
|
@ -212,9 +345,11 @@ ngx_http_ip2region_variable(ngx_http_request_t *r,
|
||||||
{
|
{
|
||||||
ngx_http_ip2region_conf_t *ip2region_conf;
|
ngx_http_ip2region_conf_t *ip2region_conf;
|
||||||
struct sockaddr_in *sin;
|
struct sockaddr_in *sin;
|
||||||
char region[512] = {'\0'};
|
char region_buffer[512] = {'\0'};
|
||||||
int err;
|
xdb_region_buffer_t region;
|
||||||
|
int err = 1;
|
||||||
unsigned int ip;
|
unsigned int ip;
|
||||||
|
xdb_searcher_t *searcher_ptr = NULL;
|
||||||
|
|
||||||
#if (NGX_HAVE_INET6)
|
#if (NGX_HAVE_INET6)
|
||||||
u_char *p;
|
u_char *p;
|
||||||
|
|
@ -224,47 +359,119 @@ ngx_http_ip2region_variable(ngx_http_request_t *r,
|
||||||
|
|
||||||
ip2region_conf = ngx_http_get_module_main_conf(r, ngx_http_ip2region_module);
|
ip2region_conf = ngx_http_get_module_main_conf(r, ngx_http_ip2region_module);
|
||||||
|
|
||||||
if (ip2region_conf->ip2region_searcher != NULL) {
|
if (ip2region_conf->v4_searcher == NULL && ip2region_conf->v6_searcher == NULL) {
|
||||||
|
v->not_found = 1;
|
||||||
|
return NGX_OK;
|
||||||
|
}
|
||||||
|
|
||||||
switch (r->connection->sockaddr->sa_family) {
|
// 初始化 region buffer
|
||||||
case AF_INET:
|
err = xdb_region_buffer_init(®ion, region_buffer, sizeof(region_buffer));
|
||||||
sin = (struct sockaddr_in *) r->connection->sockaddr;
|
if (err != 0) {
|
||||||
ip = htonl(sin->sin_addr.s_addr);
|
v->not_found = 1;
|
||||||
err = xdb_search(&ip2region_conf->ip2region_searcher->searcher, ip, region, sizeof(region));
|
return NGX_OK;
|
||||||
if (err == 0) {
|
}
|
||||||
v->data = (unsigned char *)region;
|
|
||||||
v->len = strlen(region);
|
switch (r->connection->sockaddr->sa_family) {
|
||||||
return NGX_OK;
|
case AF_INET:
|
||||||
}
|
sin = (struct sockaddr_in *) r->connection->sockaddr;
|
||||||
|
|
||||||
|
// 检查是否有IPv4 searcher
|
||||||
|
if (ip2region_conf->v4_searcher == NULL) {
|
||||||
|
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
|
||||||
|
"no IPv4 searcher available for IPv4 address");
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
searcher_ptr = &ip2region_conf->v4_searcher->searcher;
|
||||||
|
|
||||||
|
// 正确转换IP地址字节序,按ip2region期望的格式
|
||||||
|
ip = ntohl(sin->sin_addr.s_addr); // 将网络字节序转换为主机字节序
|
||||||
|
// 按照xdb_parse_v4_ip中的格式重新组织字节
|
||||||
|
{
|
||||||
|
bytes_ip_t ip_bytes[4];
|
||||||
|
ip_bytes[0] = (ip >> 24) & 0xFF;
|
||||||
|
ip_bytes[1] = (ip >> 16) & 0xFF;
|
||||||
|
ip_bytes[2] = (ip >> 8) & 0xFF;
|
||||||
|
ip_bytes[3] = ip & 0xFF;
|
||||||
|
err = xdb_search(searcher_ptr, ip_bytes, 4, ®ion);
|
||||||
|
}
|
||||||
|
if (err == 0) {
|
||||||
|
v->data = (unsigned char *)region.value;
|
||||||
|
v->len = strlen(region.value);
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
return NGX_OK;
|
||||||
|
} else {
|
||||||
|
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
|
||||||
|
"ip2region search failed for IPv4 address");
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
#if (NGX_HAVE_INET6)
|
#if (NGX_HAVE_INET6)
|
||||||
|
|
||||||
case AF_INET6:
|
case AF_INET6:
|
||||||
sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
|
sin6 = (struct sockaddr_in6 *) r->connection->sockaddr;
|
||||||
p = sin6->sin6_addr.s6_addr;
|
p = sin6->sin6_addr.s6_addr;
|
||||||
|
|
||||||
if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
|
if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
|
||||||
|
// 处理IPv4映射的IPv6地址
|
||||||
|
if (ip2region_conf->v4_searcher != NULL) {
|
||||||
|
searcher_ptr = &ip2region_conf->v4_searcher->searcher;
|
||||||
addr = p[12] << 24;
|
addr = p[12] << 24;
|
||||||
addr += p[13] << 16;
|
addr += p[13] << 16;
|
||||||
addr += p[14] << 8;
|
addr += p[14] << 8;
|
||||||
addr += p[15];
|
addr += p[15];
|
||||||
|
// 按照xdb_parse_v4_ip中的格式重新组织字节
|
||||||
ip = htonl(addr);
|
{
|
||||||
err = xdb_search(&ip2region_conf->ip2region_searcher->searcher, ip, region, sizeof(region));
|
bytes_ip_t ip_bytes[4];
|
||||||
|
ip_bytes[0] = (addr >> 24) & 0xFF;
|
||||||
|
ip_bytes[1] = (addr >> 16) & 0xFF;
|
||||||
|
ip_bytes[2] = (addr >> 8) & 0xFF;
|
||||||
|
ip_bytes[3] = addr & 0xFF;
|
||||||
|
err = xdb_search(searcher_ptr, ip_bytes, 4, ®ion);
|
||||||
|
}
|
||||||
if (err == 0) {
|
if (err == 0) {
|
||||||
v->data = (unsigned char *)region;
|
v->data = (unsigned char *)region.value;
|
||||||
v->len = strlen(region);
|
v->len = strlen(region.value);
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
return NGX_OK;
|
return NGX_OK;
|
||||||
|
} else {
|
||||||
|
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
|
||||||
|
"ip2region search failed for IPv4-mapped IPv6 address");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
} else {
|
||||||
|
// 处理纯IPv6地址
|
||||||
|
if (ip2region_conf->v6_searcher != NULL) {
|
||||||
|
searcher_ptr = &ip2region_conf->v6_searcher->searcher;
|
||||||
|
bytes_ip_t ip6_bytes[16];
|
||||||
|
if (p != NULL) {
|
||||||
|
memcpy(ip6_bytes, p, 16);
|
||||||
|
err = xdb_search(searcher_ptr, ip6_bytes, 16, ®ion);
|
||||||
|
if (err == 0) {
|
||||||
|
v->data = (unsigned char *)region.value;
|
||||||
|
v->len = strlen(region.value);
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
return NGX_OK;
|
||||||
|
} else {
|
||||||
|
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
|
||||||
|
"ip2region search failed for IPv6 address");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ngx_log_error(NGX_LOG_WARN, r->connection->log, 0,
|
||||||
|
"no IPv6 searcher available for IPv6 address");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
// 如果搜索失败,释放 region buffer
|
||||||
|
xdb_region_buffer_free(®ion);
|
||||||
|
|
||||||
|
ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
|
||||||
|
"ip2region: no region found for IP address");
|
||||||
v->not_found = 1;
|
v->not_found = 1;
|
||||||
return NGX_OK;
|
return NGX_OK;
|
||||||
}
|
}
|
||||||
|
|
@ -275,28 +482,48 @@ ngx_http_ip2region_cleanup(void *data)
|
||||||
{
|
{
|
||||||
ngx_http_ip2region_conf_t *ip2region_conf = data;
|
ngx_http_ip2region_conf_t *ip2region_conf = data;
|
||||||
|
|
||||||
if(ip2region_conf->ip2region_searcher != NULL) {
|
// 清理 IPv4 searcher
|
||||||
xdb_close(&ip2region_conf->ip2region_searcher->searcher);
|
if(ip2region_conf->v4_searcher != NULL) {
|
||||||
|
xdb_close(&ip2region_conf->v4_searcher->searcher);
|
||||||
|
|
||||||
// check and free the vector index
|
// check and free the vector index
|
||||||
if (ip2region_conf->ip2region_searcher->v_index != NULL) {
|
if (ip2region_conf->v4_searcher->v_index != NULL) {
|
||||||
xdb_close_vector_index(ip2region_conf->ip2region_searcher->v_index);
|
xdb_free_vector_index((void *)ip2region_conf->v4_searcher->v_index);
|
||||||
ip2region_conf->ip2region_searcher->v_index = NULL;
|
ip2region_conf->v4_searcher->v_index = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
// check and free the content buffer
|
// check and free the content buffer
|
||||||
if (ip2region_conf->ip2region_searcher->c_buffer != NULL) {
|
if (ip2region_conf->v4_searcher->c_buffer != NULL) {
|
||||||
xdb_close_content(ip2region_conf->ip2region_searcher->c_buffer);
|
xdb_free_content((void *)ip2region_conf->v4_searcher->c_buffer);
|
||||||
ip2region_conf->ip2region_searcher->c_buffer = NULL;
|
ip2region_conf->v4_searcher->c_buffer = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
ip2region_conf->ip2region_searcher = NULL;
|
ip2region_conf->v4_searcher = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 清理 IPv6 searcher
|
||||||
|
if(ip2region_conf->v6_searcher != NULL) {
|
||||||
|
xdb_close(&ip2region_conf->v6_searcher->searcher);
|
||||||
|
|
||||||
|
// check and free the vector index
|
||||||
|
if (ip2region_conf->v6_searcher->v_index != NULL) {
|
||||||
|
xdb_free_vector_index((void *)ip2region_conf->v6_searcher->v_index);
|
||||||
|
ip2region_conf->v6_searcher->v_index = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check and free the content buffer
|
||||||
|
if (ip2region_conf->v6_searcher->c_buffer != NULL) {
|
||||||
|
xdb_free_content((void *)ip2region_conf->v6_searcher->c_buffer);
|
||||||
|
ip2region_conf->v6_searcher->c_buffer = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
ip2region_conf->v6_searcher = NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static ngx_int_t
|
static ngx_int_t
|
||||||
ngx_http_ua_parser_test_full_name(char *name)
|
ngx_http_ip2region_is_absolute_path(char *name)
|
||||||
{
|
{
|
||||||
#if (NGX_WIN32)
|
#if (NGX_WIN32)
|
||||||
u_char c0, c1;
|
u_char c0, c1;
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,13 @@
|
||||||
#include <ngx_config.h>
|
#include <ngx_config.h>
|
||||||
#include <ngx_core.h>
|
#include <ngx_core.h>
|
||||||
#include <ngx_http.h>
|
#include <ngx_http.h>
|
||||||
#include <xdb_searcher.h>
|
#include <xdb_api.h>
|
||||||
|
|
||||||
|
#if (NGX_HAVE_INET6)
|
||||||
|
#include <sys/socket.h>
|
||||||
|
#include <netinet/in.h>
|
||||||
|
#include <arpa/inet.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
xdb_searcher_t searcher;
|
xdb_searcher_t searcher;
|
||||||
|
|
@ -17,7 +23,8 @@ typedef struct {
|
||||||
} ip2region_searcher_t;
|
} ip2region_searcher_t;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
ip2region_searcher_t *ip2region_searcher;
|
ip2region_searcher_t *v4_searcher; // IPv4 searcher for ip2region_db
|
||||||
|
ip2region_searcher_t *v6_searcher; // IPv6 searcher for ip2region_db6
|
||||||
} ngx_http_ip2region_conf_t;
|
} ngx_http_ip2region_conf_t;
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
# :cn: [中文简体]
|
||||||
|
|
||||||
|
# ip2region nodejs 查询客户端
|
||||||
|
|
||||||
|
请使用最新的 IPv6 兼容的 javascript binding:[javascript binding](../javascript/)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# :globe_with_meridians: [English]
|
||||||
|
|
||||||
|
# ip2region nodejs query client
|
||||||
|
|
||||||
|
Please use the latest IPv6-compliant JavaScript binding: [javascript binding](../javascript/)
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
# ip2region nodejs 查询客户端实现
|
|
||||||
|
|
||||||
请使用最新的 IPv6 兼容的 javascript binding:[javascript binding](../javascript/)
|
|
||||||
|
|
@ -0,0 +1,255 @@
|
||||||
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region PHP Query Client
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
### About Query API
|
||||||
|
|
||||||
|
The prototype of the Query API is as follows:
|
||||||
|
|
||||||
|
```php
|
||||||
|
// Query via string IP
|
||||||
|
// @throw Exception
|
||||||
|
search($ip_string) string
|
||||||
|
|
||||||
|
// Query via binary IP returned by Util.parseIP
|
||||||
|
// @throw Exception
|
||||||
|
searchByBytes($ip_bytes) string
|
||||||
|
```
|
||||||
|
|
||||||
|
If the query fails, an exception will be thrown; if the query is successful, the `region` information string will be returned; if the IP being queried cannot be found, an empty string `""` will be returned.
|
||||||
|
|
||||||
|
### About IPv4 and IPv6
|
||||||
|
|
||||||
|
This xdb query client implementation supports both IPv4 and IPv6 queries. The usage is as follows:
|
||||||
|
|
||||||
|
```php
|
||||||
|
use \ip2region\xdb\{IPv4, IPv6};
|
||||||
|
|
||||||
|
// For IPv4: Set xdb path to the v4 xdb file, specify IP version as IPv4
|
||||||
|
$dbFile = "../../data/ip2region_v4.xdb"; // or your ipv4 xdb path
|
||||||
|
$version = IPv4::default();
|
||||||
|
|
||||||
|
// For IPv6: Set xdb path to the v6 xdb file, specify IP version as IPv6
|
||||||
|
$dbFile = "../../data/ip2region_v6.xdb"; // or your ipv6 xdb path
|
||||||
|
$version = IPv6::default();
|
||||||
|
|
||||||
|
// The IP version of the xdb specified by dbPath must be consistent with the version specified, otherwise an error will occur during query execution
|
||||||
|
// Note: The following demonstration directly uses $dbFile and $version variables
|
||||||
|
```
|
||||||
|
|
||||||
|
### XDB File Verification
|
||||||
|
|
||||||
|
It is recommended that you proactively verify the applicability of the xdb file, as some new features in the future may cause the current Searcher version to be incompatible with the xdb file you are using. Verification can avoid unpredictable errors during runtime. You do not need to verify every time; for example, verify when the service starts or manually call the command to confirm version matching. Do not run verification every time a Searcher is created, as this will affect query response speed, especially in high-concurrency scenarios.
|
||||||
|
|
||||||
|
```php
|
||||||
|
use \ip2region\xdb\Util;
|
||||||
|
|
||||||
|
$err = Util::verify($dbFile);
|
||||||
|
if ($err != null) {
|
||||||
|
// Applicability verification failed!!!
|
||||||
|
// The current query client implementation is not suitable for querying the xdb file specified by dbPath.
|
||||||
|
// You should stop the service and use a suitable xdb file or upgrade to a Searcher implementation compatible with dbPath.
|
||||||
|
printf("failed to verify xdb file `%s`: %s\n", $dbFile, $err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verification passed, the current Searcher can be safely used for query operations on the xdb pointed to by dbPath
|
||||||
|
```
|
||||||
|
|
||||||
|
### File-Based Query
|
||||||
|
|
||||||
|
```php
|
||||||
|
// require or autoload the xdb\Searcher
|
||||||
|
require 'xdb\Searcher.php';
|
||||||
|
use \ip2region\xdb\Util;
|
||||||
|
use \ip2region\xdb\Searcher;
|
||||||
|
|
||||||
|
// 1. Create a Searcher object using the $version and $dbFile mentioned above
|
||||||
|
try {
|
||||||
|
$searcher = Searcher::newWithFileOnly($version, $dbFile);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
printf("failed to create searcher with '%s': %s\n", $dbFile, $e->getMessage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Query, both IPv4 or IPv6 addresses are supported
|
||||||
|
try {
|
||||||
|
$ip = '1.2.3.4';
|
||||||
|
// $ip = ""240e:3b7:3272:d8d0:db09:c067:8d59:539e; // IPv6
|
||||||
|
$sTime = Util::now();
|
||||||
|
$region = $searcher->search($ip);
|
||||||
|
$costMs = Util::now() - $sTime;
|
||||||
|
printf("{region: %s, took: %.5f ms}\n", $region, $costMs);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
printf("failed to search(%s): %s", $ip, $e->getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Close resources
|
||||||
|
$searcher->close();
|
||||||
|
|
||||||
|
// Note: For concurrent use, each thread or coroutine needs to create an independent searcher object.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching `VectorIndex`
|
||||||
|
|
||||||
|
If your PHP environment supports it, you can pre-load the vectorIndex cache and make it a global variable. Using the global vectorIndex every time you create a Searcher can reduce a fixed IO operation, thereby accelerating queries and reducing IO pressure.
|
||||||
|
|
||||||
|
```php
|
||||||
|
// require or autoload the xdb\Searcher
|
||||||
|
require 'xdb\Searcher.php';
|
||||||
|
use \ip2region\xdb\Util;
|
||||||
|
use \ip2region\xdb\Searcher;
|
||||||
|
|
||||||
|
|
||||||
|
// 1. Load VectorIndex cache from $dbFile and cache the following vIndex variable in memory.
|
||||||
|
$vIndex = Util::loadVectorIndexFromFile($dbFile);
|
||||||
|
if ($vIndex === null) {
|
||||||
|
printf("failed to load vector index from '%s'\n", $dbFile);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Use the global vIndex to create a query object with VectorIndex cache.
|
||||||
|
try {
|
||||||
|
$searcher = Searcher::newWithVectorIndex($version, $dbFile, $vIndex);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
printf("failed to create vectorIndex searcher with '%s': %s\n", $dbFile, $e->getMessage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Query, both IPv4 or IPv6 are supported
|
||||||
|
try {
|
||||||
|
$ip = '1.2.3.4';
|
||||||
|
// $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
$sTime = Util::now();
|
||||||
|
$region = $searcher->search($ip);
|
||||||
|
$costMs = Util::now() - $sTime;
|
||||||
|
printf("{region: %s, took: %.5f ms}\n", $region, $costMs);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
printf("failed to search(%s): %s", $ip, $e->getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Close resources
|
||||||
|
$searcher->close();
|
||||||
|
|
||||||
|
// Note: For concurrent use, each thread or coroutine needs to create an independent searcher object, but they all share the same read-only global vectorIndex.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Caching the Entire `xdb` File
|
||||||
|
|
||||||
|
If your PHP environment supports it, you can pre-load the entire `xdb` file into memory. This allows for fully memory-based queries, similar to the previous memory search.
|
||||||
|
|
||||||
|
```php
|
||||||
|
// require or autoload the xdb\Searcher
|
||||||
|
require 'xdb\Searcher.php';
|
||||||
|
use \ip2region\xdb\Util;
|
||||||
|
use \ip2region\xdb\Searcher;
|
||||||
|
|
||||||
|
// 1. Load the entire xdb from $dbFile into memory.
|
||||||
|
$cBuff = Util::loadContentFromFile($dbFile);
|
||||||
|
if ($cBuff === null) {
|
||||||
|
printf("failed to load content buffer from '%s'\n", $dbFile);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Use the global cBuff to create a query object that is fully based on memory.
|
||||||
|
try {
|
||||||
|
$searcher = Searcher::newWithBuffer($version, $cBuff);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
printf("failed to create buffer cached searcher: %s\n", $dbFile, $e->getMessage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Query, both IPv4 or IPv6 are supported
|
||||||
|
try {
|
||||||
|
$ip = '1.2.3.4';
|
||||||
|
// $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
|
$sTime = Util::now();
|
||||||
|
$region = $searcher->search($ip);
|
||||||
|
$costMs = Util::now() - $sTime;
|
||||||
|
printf("{region: %s, took: %.5f ms}\n", $region, $costMs);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
printf("failed to search(%s): %s", $ip, $e->getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Close resources
|
||||||
|
// This searcher object can be safely used for concurrency; close it when the entire service is shut down
|
||||||
|
// $searcher->close();
|
||||||
|
|
||||||
|
// Note: For concurrent use, the searcher object created with the entire xdb cache can be safely used for concurrency.
|
||||||
|
```
|
||||||
|
|
||||||
|
# Query Testing
|
||||||
|
|
||||||
|
Run query tests via the `search_test.php` script:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ php git:(fr_php_ipv6) ✗ php search_test.php
|
||||||
|
php search_test.php [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: using the default data/ip2region_v4.xdb for IPv4 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ php git:(fr_php_ipv6) ✗ php search_test.php --db=../../data/ip2region_v4.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb file: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 0.12695 ms}
|
||||||
|
ip2region>> 120.229.45.2
|
||||||
|
{region: 中国|广东省|深圳市|移动|CN, ioCount: 3, took: 0.07397 ms}
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: using the default data/ip2region_v6.xdb for IPv6 query testing:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ php git:(master) ✗ php ./search_test.php --db=../../data/ip2region_v6.xdb
|
||||||
|
ip2region xdb searcher test program
|
||||||
|
source xdb file: ../../data/ip2region_v6.xdb (IPv6, vectorIndex)
|
||||||
|
type 'quit' to exit
|
||||||
|
ip2region>> ::
|
||||||
|
{region: , ioCount: 1, took: 0.08887 ms}
|
||||||
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
|
{region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 0.10303 ms}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, ioCount: 13, took: 0.04614 ms}
|
||||||
|
```
|
||||||
|
|
||||||
|
Enter an IP to perform a query test. You can also set `cache-policy` to file/vectorIndex/content respectively to test the efficiency of the three different cache implementations.
|
||||||
|
|
||||||
|
# Bench Testing
|
||||||
|
|
||||||
|
Run automatic bench testing via the `bench_test.php` script. On one hand, this ensures that there are no errors in the `xdb` file; on the other hand, it tests average query performance through a large number of queries:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
➜ php git:(fr_php_ipv6) ✗ php bench_test.php
|
||||||
|
php bench_test.php [command options]
|
||||||
|
options:
|
||||||
|
--db string ip2region binary xdb file path
|
||||||
|
--src string source ip text file path
|
||||||
|
--cache-policy string cache policy: file/vectorIndex/content
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: perform an IPv4 bench test using the default data/ip2region_v4.xdb and data/ipv4_source.txt files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php bench_test.php --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
For example: perform an IPv6 bench test using the default data/ip2region_v6.xdb and data/ipv6_source.txt files:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
php bench_test.php --db=../../data/ip2region_v6.xdb --src=../../data/ipv6_source.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
You can test the performance of the three different cache implementations (file/vectorIndex/content) by setting the `cache-policy` parameter.
|
||||||
|
@Note: Please ensure that the src file used for benching is the same source file used to generate the corresponding xdb file.
|
||||||
|
|
||||||
|
### Third-party Repository Support
|
||||||
|
|
||||||
|
1. Composer supported [zoujingli/ip2region](https://github.com/zoujingli/ip2region) - IPv6 supported.
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
# ip2region xdb php 查询客户端实现
|
:globe_with_meridians: [中文简体](README_zh.md) | [English](README.md)
|
||||||
|
|
||||||
|
# ip2region php 查询客户端
|
||||||
|
|
||||||
# 使用方式
|
# 使用方式
|
||||||
|
|
||||||
|
|
@ -68,7 +70,7 @@ try {
|
||||||
// 2, 查询,IPv4 或者 IPv6 的地址都支持
|
// 2, 查询,IPv4 或者 IPv6 的地址都支持
|
||||||
try {
|
try {
|
||||||
$ip = '1.2.3.4';
|
$ip = '1.2.3.4';
|
||||||
// $ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// $ip = ""240e:3b7:3272:d8d0:db09:c067:8d59:539e; // IPv6
|
||||||
$sTime = Util::now();
|
$sTime = Util::now();
|
||||||
$region = $searcher->search($ip);
|
$region = $searcher->search($ip);
|
||||||
$costMs = Util::now() - $sTime;
|
$costMs = Util::now() - $sTime;
|
||||||
|
|
@ -111,7 +113,7 @@ try {
|
||||||
// 3、查询,IPv4 或者 IPv6 都支持
|
// 3、查询,IPv4 或者 IPv6 都支持
|
||||||
try {
|
try {
|
||||||
$ip = '1.2.3.4';
|
$ip = '1.2.3.4';
|
||||||
// $ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
$sTime = Util::now();
|
$sTime = Util::now();
|
||||||
$region = $searcher->search($ip);
|
$region = $searcher->search($ip);
|
||||||
$costMs = Util::now() - $sTime;
|
$costMs = Util::now() - $sTime;
|
||||||
|
|
@ -126,7 +128,7 @@ $searcher->close();
|
||||||
// 备注:并发使用,每个线程或者协程需要创建一个独立的 searcher 对象,但是都共享统一的只读全局 vectorIndex。。
|
// 备注:并发使用,每个线程或者协程需要创建一个独立的 searcher 对象,但是都共享统一的只读全局 vectorIndex。。
|
||||||
```
|
```
|
||||||
|
|
||||||
### 缓存整个 `xdb` 数据
|
### 缓存整个 `xdb` 文件
|
||||||
|
|
||||||
如果你的 PHP 母环境支持,可以预先加载整个 `xdb` 的数据到内存,这样可以实现完全基于内存的查询,类似之前的 memory search 查询。
|
如果你的 PHP 母环境支持,可以预先加载整个 `xdb` 的数据到内存,这样可以实现完全基于内存的查询,类似之前的 memory search 查询。
|
||||||
```php
|
```php
|
||||||
|
|
@ -153,7 +155,7 @@ try {
|
||||||
// 3、查询,IPv4 或者 IPv6 都支持
|
// 3、查询,IPv4 或者 IPv6 都支持
|
||||||
try {
|
try {
|
||||||
$ip = '1.2.3.4';
|
$ip = '1.2.3.4';
|
||||||
// $ip = "2001:4:112:ffff:ffff:ffff:ffff:ffff"; // IPv6
|
// $ip = "240e:3b7:3272:d8d0:db09:c067:8d59:539e"; // IPv6
|
||||||
$sTime = Util::now();
|
$sTime = Util::now();
|
||||||
$region = $searcher->search($ip);
|
$region = $searcher->search($ip);
|
||||||
$costMs = Util::now() - $sTime;
|
$costMs = Util::now() - $sTime;
|
||||||
|
|
@ -186,8 +188,10 @@ options:
|
||||||
ip2region xdb searcher test program
|
ip2region xdb searcher test program
|
||||||
source xdb file: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
source xdb file: ../../data/ip2region_v4.xdb (IPv4, vectorIndex)
|
||||||
type 'quit' to exit
|
type 'quit' to exit
|
||||||
|
ip2region>> 1.2.3.4
|
||||||
|
{region: Australia|Queensland|Brisbane|0|AU, ioCount: 5, took: 0.12695 ms}
|
||||||
ip2region>> 120.229.45.2
|
ip2region>> 120.229.45.2
|
||||||
{region: 中国|广东省|深圳市|移动, ioCount: 3, took: 0.02783 ms}
|
{region: 中国|广东省|深圳市|移动|CN, ioCount: 3, took: 0.07397 ms}
|
||||||
```
|
```
|
||||||
|
|
||||||
例如:使用默认的 data/ip2region_v6.xdb 进行 IPv6 的查询测试:
|
例如:使用默认的 data/ip2region_v6.xdb 进行 IPv6 的查询测试:
|
||||||
|
|
@ -199,7 +203,9 @@ type 'quit' to exit
|
||||||
ip2region>> ::
|
ip2region>> ::
|
||||||
{region: , ioCount: 1, took: 0.08887 ms}
|
{region: , ioCount: 1, took: 0.08887 ms}
|
||||||
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
ip2region>> 240e:3b7:3272:d8d0:db09:c067:8d59:539e
|
||||||
{region: 中国|广东省|深圳市|家庭宽带, ioCount: 8, took: 0.09595 ms}
|
{region: 中国|广东省|深圳市|电信|CN, ioCount: 8, took: 0.10303 ms}
|
||||||
|
ip2region>> 2604:a840:3::a04d
|
||||||
|
{region: United States|California|San Jose|xTom|US, ioCount: 13, took: 0.04614 ms}
|
||||||
```
|
```
|
||||||
|
|
||||||
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
||||||
|
|
@ -144,6 +144,14 @@ while (!feof($handle)) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//@Note extract the ip address from line with special chars
|
||||||
|
// if (preg_match('/^([0-9\.]+)/', $line, $m) != 1) {
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
|
||||||
|
// $line = $m[1];
|
||||||
|
// print("ip: {$m[1]}\n");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$ipBytes = Util::parseIP($line);
|
$ipBytes = Util::parseIP($line);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
|
|
||||||
|
|
@ -504,6 +504,12 @@ class Searcher {
|
||||||
}
|
}
|
||||||
|
|
||||||
// printf("sPtr: %d, ePtr: %d\n", $sPtr, $ePtr);
|
// printf("sPtr: %d, ePtr: %d\n", $sPtr, $ePtr);
|
||||||
|
// @Note: ptr validate, zero ptr means source data missing
|
||||||
|
// so we could just stop here and return an empty string.
|
||||||
|
if ($sPtr == 0 || $ePtr == 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
[$bytes, $dBytes] = [strlen($ipBytes), strlen($ipBytes) << 1];
|
[$bytes, $dBytes] = [strlen($ipBytes), strlen($ipBytes) << 1];
|
||||||
|
|
||||||
// binary search the segment index to get the region info
|
// binary search the segment index to get the region info
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue