fix make curvebs image

Signed-off-by: Hanqing Wu <wuhanqing@corp.netease.com>

.bazelrc

@@ -3,8 +3,8 @@ build --verbose_failures
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
-build:gcc7-later --cxxopt -faligned-new
-build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
 
+build:gcc7-later --cxxopt -faligned-new
+
 run --copt=-fdiagnostics-color=always

.bazelversion

@@ -1 +1 @@
-4.2.2
+0.17.2

.clang-format

@@ -1,147 +0,0 @@
----
-Language:        Cpp
-AccessModifierOffset: -3
-AlignAfterOpenBracket: Align
-AlignConsecutiveMacros: true
-AlignConsecutiveAssignments: false
-AlignConsecutiveBitFields: false
-AlignConsecutiveDeclarations: false
-AlignEscapedNewlines: Right
-AlignOperands:   Align
-AlignTrailingComments: true
-AllowAllArgumentsOnNextLine: true
-AllowAllConstructorInitializersOnNextLine: true
-AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortEnumsOnASingleLine: true
-AllowShortBlocksOnASingleLine: Never
-AllowShortCaseLabelsOnASingleLine: false
-AllowShortFunctionsOnASingleLine: All
-AllowShortLambdasOnASingleLine: All
-AllowShortIfStatementsOnASingleLine: Never
-AllowShortLoopsOnASingleLine: false
-AlwaysBreakAfterDefinitionReturnType: None
-AlwaysBreakAfterReturnType: None
-AlwaysBreakBeforeMultilineStrings: false
-AlwaysBreakTemplateDeclarations: MultiLine
-BinPackArguments: true
-BinPackParameters: true
-BraceWrapping:
-  AfterCaseLabel:  false
-  AfterClass:      false
-  AfterControlStatement: Never
-  AfterEnum:       false
-  AfterFunction:   false
-  AfterNamespace:  false
-  AfterObjCDeclaration: false
-  AfterStruct:     false
-  AfterUnion:      false
-  AfterExternBlock: false
-  BeforeCatch:     false
-  BeforeElse:      false
-  BeforeLambdaBody: false
-  BeforeWhile:     false
-  IndentBraces:    false
-  SplitEmptyFunction: true
-  SplitEmptyRecord: true
-  SplitEmptyNamespace: true
-BreakBeforeBinaryOperators: None
-BreakBeforeBraces: Attach
-BreakBeforeInheritanceComma: false
-BreakInheritanceList: BeforeColon
-BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: false
-BreakConstructorInitializers: BeforeColon
-BreakAfterJavaFieldAnnotations: false
-BreakStringLiterals: true
-ColumnLimit:     80
-CommentPragmas:  '^ IWYU pragma:'
-CompactNamespaces: false
-ConstructorInitializerAllOnOneLineOrOnePerLine: false
-ConstructorInitializerIndentWidth: 4
-ContinuationIndentWidth: 4
-Cpp11BracedListStyle: true
-DeriveLineEnding: true
-DerivePointerAlignment: false
-DisableFormat:   false
-ExperimentalAutoDetectBinPacking: false
-FixNamespaceComments: true
-ForEachMacros:
-  - foreach
-  - Q_FOREACH
-  - BOOST_FOREACH
-IncludeBlocks:   Preserve
-IncludeCategories:
-  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
-    Priority:        2
-    SortPriority:    0
-  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
-    Priority:        3
-    SortPriority:    0
-  - Regex:           '.*'
-    Priority:        1
-    SortPriority:    0
-IncludeIsMainRegex: '(Test)?$'
-IncludeIsMainSourceRegex: ''
-IndentCaseLabels: false
-IndentCaseBlocks: false
-IndentGotoLabels: true
-IndentPPDirectives: None
-IndentExternBlock: AfterExternBlock
-IndentWidth:     4
-IndentWrappedFunctionNames: false
-InsertTrailingCommas: None
-JavaScriptQuotes: Leave
-JavaScriptWrapImports: true
-KeepEmptyLinesAtTheStartOfBlocks: true
-MacroBlockBegin: ''
-MacroBlockEnd:   ''
-MaxEmptyLinesToKeep: 2
-NamespaceIndentation: None
-ObjCBinPackProtocolList: Auto
-ObjCBlockIndentWidth: 2
-ObjCBreakBeforeNestedBlockParam: true
-ObjCSpaceAfterProperty: false
-ObjCSpaceBeforeProtocolList: true
-PenaltyBreakAssignment: 2
-PenaltyBreakBeforeFirstCallParameter: 19
-PenaltyBreakComment: 300
-PenaltyBreakFirstLessLess: 120
-PenaltyBreakString: 1000
-PenaltyBreakTemplateDeclaration: 10
-PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 60
-PointerAlignment: Right
-ReflowComments:  true
-SortIncludes:    false
-SortUsingDeclarations: true
-SpaceAfterCStyleCast: false
-SpaceAfterLogicalNot: false
-SpaceAfterTemplateKeyword: true
-SpaceBeforeAssignmentOperators: true
-SpaceBeforeCpp11BracedList: false
-SpaceBeforeCtorInitializerColon: true
-SpaceBeforeInheritanceColon: true
-SpaceBeforeParens: ControlStatements
-SpaceBeforeRangeBasedForLoopColon: true
-SpaceInEmptyBlock: false
-SpaceInEmptyParentheses: false
-SpacesBeforeTrailingComments: 2
-SpacesInAngles:  false
-SpacesInConditionalStatement: false
-SpacesInContainerLiterals: true
-SpacesInCStyleCastParentheses: false
-SpacesInParentheses: false
-SpacesInSquareBrackets: false
-SpaceBeforeSquareBrackets: false
-Standard:        Cpp11
-StatementMacros:
-  - Q_UNUSED
-  - QT_REQUIRE_VERSION
-TabWidth:        4
-UseCRLF:         false
-UseTab:          Never
-WhitespaceSensitiveMacros:
-  - STRINGIZE
-  - PP_STRINGIZE
-  - BOOST_PP_STRINGIZE
-...

.clangd

@@ -1,12 +0,0 @@
-Diagnostics:
-  ClangTidy:
-    Add: [performance-*, modernize-*, readability-*,bugprone-*,]
-    Remove: [modernize-use-trailing-return-type]
-    CheckOptions:
-      readability-identifier-naming.VariableCase: camelBack
-  UnusedIncludes: Strict
-
-InlayHints:
-  Enabled: No
-  ParameterNames: No
-  DeducedTypes: No

.github/ISSUE_TEMPLATE/bug_report.md

@@ -19,7 +19,11 @@ assignees: ''
 **Versions (各种版本)**
 OS:
 Compiler:
-branch:
-commit id:
+curve-mds:
+curve-chunkserver:
+curve-snapshotcloneserver:
+curve-sdk:
+nebd:
+curve-nbd:
 
 **Additional context/screenshots (更多上下文/截图)**

.github/pull_request_template.md

@@ -2,7 +2,7 @@
 
 ### What problem does this PR solve?
 
-Issue Number: #xxx <!-- replace xxx with issue number -->
+Issue Number: close #xxx <!-- REMOVE this line if no issue to close -->
 
 Problem Summary:
 
@@ -12,7 +12,7 @@ What's Changed:
 
 How it Works:
 
-Side effects(Breaking backward compatibility? Performance regression?):
+Side effects(Breaking backward compatibility? Performance regression?): 
 
 ### Check List
 

.github/workflows/issue-action.yml

@@ -1,16 +0,0 @@
-name: Issue assignment
-
-on:
-    issues:
-        types: [opened]
-
-jobs:
-    auto-assign:
-        runs-on: ubuntu-latest
-        steps:
-            - name: 'Auto-assign issue'
-              uses: pozil/auto-assign-issue@v1.10.1
-              with:
-                  assignees: cw123,ilixiaocui,wuhongsong,Cyber-SiKu,jolly-sy
-                  numOfAssignee: 1
-                  allowSelfAssign: true

.gitignore

@@ -35,7 +35,6 @@ CMakeLists.txt
 bin
 *.temp
 .clwb
-*.swp
 
 # cscope
 cscope.out
@@ -72,13 +71,8 @@ bazel-testlogs
 
 #log
 *.log
-runlog/
-
-# history
-.gdb_history
 
 .clang-format
-.clangd
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
@@ -91,7 +85,6 @@ test/client/configs/*
 projects/*
 docker/curvebs
 docker/base/*
-docker/debian*/*so*
 !docker/base/Dockerfile
 !docker/base/Makefile
 
@@ -103,61 +96,20 @@ curvefs/BUILD_MODE
 *.pyc
 .facts/
 *retry
-# monitor
-curvefs/monitor/prometheus/target.json
 
-curvefs/docker/*/curvefs
 curvefs/docker/curvefs
 curvefs/docker/base/*
-curvefs/docker/debian*/*so*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
 curvefs/BUILD_MODE
 
 .BUILD_MODE
-__not_found__
-
-# thirdparties
-thirdparties/rocksdb/lib/
-thirdparties/rocksdb/include/
-thirdparties/rocksdb/rocksdb/
-thirdparties/rocksdb/*log
-thirdparties/rocksdb/*.tar.gz
-thirdparties/aws/*.tar.gz
-thirdparties/etcdclient/tmp/
-thirdparties/etcdclient/*.h
-thirdparties/memcache/*.tar.gz
-thirdparties/memcache/libmemcached-*/
 
 /external
 /bazel-*
 /compile_commands.json
 /.cache/
 
-docker/curvebs
-docker/*/curvebs
-
-curvefs/docker/curvefs
-curvefs/docker/*/curvefs
-storage_*
-
-tools-v2/sbin/*
-tools-v2/proto/proto/*
-tools-v2/proto/curvefs/*
-tools-v2/*/*.test
-tools-v2/__debug_bin
-tools-v2/vendor/
-
-.test
-.note
-.playground
-.dumpfile
-metastore_test.dat
-GPATH
-GRTAGS
-GTAGS
-core.*
-
 test/integration/*.conf
 test/integration/client/config/client.conf*
 test/integration/snapshotcloneserver/config/*.conf

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "nbd"]
-	path = nbd
-	url = https://github.com/opencurve/curve-nbd

.obm.cfg

@@ -1,2 +0,0 @@
-container_name: curve-build-playground-master
-container_image: opencurvedocker/curve-base:build-debian9

BUILD

@@ -1,39 +0,0 @@
-#
-#  Copyright (c) 2020 NetEase Inc.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-#
-# Copyright 2017 The Abseil Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-package(default_visibility = ["//visibility:public"])
-
-config_setting(
-    name = "clang_compiler",
-    flag_values = {
-        "@bazel_tools//tools/cpp:compiler": "clang",
-    },
-)

CHANGELOG-1.0.md

@@ -1,5 +1,5 @@
 See [code changes](https://github.com/opencurve/curve/compare/v0.1.4...release1.0).
-# CHANGELOG of v1.0.0
+
  ## The new features:
 - Support external ip for ChunkServer to divide inter-cluster communication and communication with clients.
 - The Curve_ops_tool supports printing space-info for each logical pool.
@@ -16,30 +16,3 @@ See [code changes](https://github.com/opencurve/curve/compare/v0.1.4...release1.
 - Fixed curvefs_tool (curve) missing support for password.
 - Fixed the bug that data from Etcd would not placed in LRU cache.
 - Fixed ChunkServer not aborting when read returns an internal error.
-
-
-<hr/>
-
-# CHANGELOG of v1.0.3-rc0
-
-## New Feature
-
-- [change max file length from 4TB to 20TB](https://github.com/opencurve/curve/pull/321)
-
-
-<hr/>
-
-# CHANGELOG of v1.0.4
-
-## New Feature
-
-- [support nbd auto map at boot](https://github.com/opencurve/curve/pull/347)
-
-
-<hr/>
-
-# CHANGELOG of v1.0.5
-
-## Optimization
-
-- [Change auto map service to systemd standard](https://github.com/opencurve/curve/pull/695)

CHANGELOG-1.2.md

@@ -22,35 +22,6 @@ Previous change logs can be found at [CHANGELOG-1.1](https://github.com/opencurv
     - [Support to list the volume on copyset where majority copy is failed.](https://github.com/opencurve/curve/pull/233)
 - [Clean up the unit test temporary folder.](https://github.com/opencurve/curve/pull/206/)
 
-
-Hardware: 6 nodes, each with:
- - 20x SATA SSD Intel® SSD DC S3500 Series 800G
- - 2x Intel(R) Xeon(R) CPU E5-2660 v4 @ 2.00GHz
- - 2x Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection, bond mode is 802.3ad with layer2+3 hash policy
- - 251G RAM
-
-Performance test is based on curve-nbd, the size of each block device is 200GB, all configurations are default, and each Chunkserver is deployed on one SSD.
-
-1 NBD block device:
-
-| item | iops/bandwidth | avg-latency | 99th-latency  | striped volume<br>iops/bandwidth | striped volume<br>avg-latency | striped volume<br>99th-latency |
-| :----: | :----: | :----: | :----: | :----: |:----: |:----: |
-| 4K randwrite, 128 depth | 97,700 iops | 1.3 ms | 2.0 ms | 97,100 iops | 1.3 ms | 3.0 ms |
-| 4K randread, 128 depth | 119,000 iops | 1.07 ms | 1.8 ms | 98,000 iops | 1.3 ms | 2.2 ms |
-| 512K write, 128 depth | 208 MB/s | 307 ms | 347 ms | 462 MB/s | 138 ms | 228 ms |
-| 512K read, 128 depth | 311 MB/s | 206 ms | 264 ms | 843 MB/s | 75 ms | 102 ms |
-
-
-10 NBD block device:
-
-| item | iops/bandwidth | avg-latency | 99th-latency | striped volume<br>iops/bandwidth | striped volume<br>avg-latency | striped volume<br>99th-latency |
-| :----: | :----: | :----: | :----: | :----: |:----: |:----: |
-| 4K randwrite, 128 depth | 231,000 iops | 5.6 ms | 50 ms | 227,000 iops | 5.9 ms | 53 ms |
-| 4K randread, 128 depth | 350,000 iops | 3.7 ms | 8.2 ms | 345,000 iops | 3.8 ms | 8.2 ms |
-| 512K write, 128 depth | 805 MB/s | 415 ms | 600 ms | 1,077 MB/s | 400 ms | 593 ms |
-| 512K read, 128 depth | 2,402 MB/s | 267 ms | 275 ms | 3,313 MB/s | 201 ms | 245 ms |
-
-
 ## bug fix
 
 - [Fix clone delete bug.](https://github.com/opencurve/curve/pull/176)
@@ -60,39 +31,3 @@ Performance test is based on curve-nbd, the size of each block device is 200GB,
 - [Fixed the direct_fd leak problem when wal write disk.](https://github.com/opencurve/curve/pull/113)
 - [Fixed the atomicity problem of GetFile when not using the file pool.](https://github.com/opencurve/curve/pull/195)
 - [Fixed in cluster_basic_test mds start error on InitEtcdClient() which produced coredump](https://github.com/opencurve/curve/pull/205)
-
-
-<hr/>
-
-# CHANGELOG of v1.2.1-rc0
-
-## New Feature
-
-- [curve-nbd: support simulate 512-byte logical/physical block device](https://github.com/opencurve/curve/pull/444)
-- [curve-nbd: prevent mounted device to unmap](https://github.com/opencurve/curve/pull/453)
-- [curve-nbd: add auto mount options](https://github.com/opencurve/curve/pull/473)
-- [nebd: support multi-instances for different user](https://github.com/opencurve/curve/pull/444)
-- [client: retry allocate segment until success](https://github.com/opencurve/curve/pull/452)
-- [mds: schedule pending online chunkserver](https://github.com/opencurve/curve/pull/462)
-
-## Bug Fix
-
-- [ansible: backup and recover curvetab](https://github.com/opencurve/curve/pull/484)
-- [debian package: backup and recover curvetab](https://github.com/opencurve/curve/pull/491)
-- [scripts: fix potential overflow when cnovert ip to value](https://github.com/opencurve/curve/pull/474)
-- [client: fix segment in SourceReader](https://github.com/opencurve/curve/pull/474)
-- [client: explicit stop LeaseExecutor in FileInstance::UnInitialize](https://github.com/opencurve/curve/pull/474)
-- [curve-nbd: fix concurrent nbd map](https://github.com/opencurve/curve/pull/447)
-- [snapshotclone: fix cancel task lost](https://github.com/opencurve/curve/pull/463)
-- [tools: remove copyset data after remove peer](https://github.com/opencurve/curve/pull/458)
-- [mds: modify stale heartbeat update error with warning](https://github.com/opencurve/curve/pull/480)
-- [chunkserver: fix config change epoch update error](https://github.com/opencurve/curve/pull/487)
-
-
-<hr/>
-
-# CHANGELOG of v1.2.2
-
-## Optimization
-
-- [Change auto map service to systemd standard](https://github.com/opencurve/curve/pull/699)

CHANGELOG-1.3.md

@@ -1,61 +0,0 @@
-# CHANGELOG of v1.3
-
-Previous change logs can be found at [CHANGELOG-1.2](https://github.com/opencurve/curve/blob/master/CHANGELOG-1.2.md)
-
-## new features
-
-- [Support discard to garbage collection.](https://github.com/opencurve/curve/pull/189)
-- [QoS of client.](https://github.com/opencurve/curve/pull/268)
-- [QoS of SnapshotCloneServer.](https://github.com/opencurve/curve/pull/303)
-- [Recover file in recycleBin.](https://github.com/opencurve/curve/pull/259)
-- [Fill chunks in chunkfile pool with 0 in the background.](https://github.com/opencurve/curve/pull/322)
-- [Silent data CRC verification in the background.](https://github.com/opencurve/curve/pull/377)
-- [Periodic automatic cleaning of recyclebin.](https://github.com/opencurve/curve/pull/310)
-
-
-## optimization
-
-- Optimize optimize build script, optimize log printing.
-- Translate some document and code comment from Chinese to English.
-- [Mark a online chunkserver to pendding status to migrate it's data.](https://github.com/opencurve/curve/pull/252)
-- [Clean up the remaining copyset after remove peer.](https://github.com/opencurve/curve/pull/373)
-- [Nbd auto map at reboot.](https://github.com/opencurve/curve/pull/347)
-- ansible script improve:
-    - [Improve bool type var and some issues.](https://github.com/opencurve/curve/pull/331)
-    - [Improve daemon restart.](https://github.com/opencurve/curve/pull/315)
-    - [Incorporate some scripts in debain package and ansible template.](https://github.com/opencurve/curve/pull/380)
-
-
-## bug fix
-
-- [Fix recyclebin space statistics error bug.](https://github.com/opencurve/curve/pull/294)
-- [Fix a metric bug when not use walpool.](https://github.com/opencurve/curve/pull/291)
-- [Fix a nbd map concurrently bug.](https://github.com/opencurve/curve/pull/302)
-- [Fix discard and flatten concurrency issues.](https://github.com/opencurve/curve/pull/312)
-- [Fix max chunk id calculation error when use chunkfile poo as wal pool.](https://github.com/opencurve/curve/pull/341)
-- [Fix client retry when allocate segment failed as cluster space is pool.](https://github.com/opencurve/curve/pull/338)
-- [Add protection for unmap operation when io is not stopped.](https://github.com/opencurve/curve/pull/348)
-- [Fix a segment fault when client read from clone source.](https://github.com/opencurve/curve/pull/358)
-
-<hr>
-
-# CHANGELOG of v1.3-beta1
-
-## bug fix
-
-- [Fix duplicate delete segment in mds on discard feature.](https://github.com/opencurve/curve/pull/421)
-- [Fix trash metirc statistic bug.](https://github.com/opencurve/curve/pull/417)
-- [Fix copyset status judgment after set pendding on online statue.](https://github.com/opencurve/curve/pull/416)
-- [Add mount options in auto map nbd volume script.](https://github.com/opencurve/curve/pull/407)
-- [Fix add qos for clone volume.](https://github.com/opencurve/curve/pull/421)
-- [Fix crc issues in chunkserver.](https://github.com/opencurve/curve/pull/405)
-- [Fix crc issues in mds and add scan-status support in curve_ops_tool.](https://github.com/opencurve/curve/pull/420)
-- [Fix core dump when chunkserver SIGINT exit sometimes.](https://github.com/opencurve/curve/pull/429)
-
-<hr>
-
-# CHANGELOG of v1.3-beta2
-
-## new feature
-
-- [Support 512 bytes aligned IO requests.](https://github.com/opencurve/curve/pull/435)

CHANGELOG-2.0.md

@@ -1,26 +0,0 @@
-# CHANGELOG of v2.0
-
-The content of Curve v2.0 includes CurveBS v1.3, CurveFS v0.2.0 and some other content listed below.
-Previous change logs can be found at [CHANGELOG-1.3](https://github.com/opencurve/curve/blob/master/CHANGELOG-1.3.md)
-
-## new features
-
-- [CurveFS: support replace a server which doesn't work anymore.](https://github.com/opencurve/curve/pull/954)
-- [CurveFS: support internal and external services.](https://github.com/opencurve/curve/issues/973)
-
-
-## optimization
-
-- [CurveFS client: update inode asynchronously.](https://github.com/opencurve/curve/pull/1020)
-- [Turn off batch delete object in the config file.](https://github.com/opencurve/curve/pull/997)
-- [Remove create filesystem at mount process and use curvefs-tool instead.](https://github.com/opencurve/curve/pull/899)
-- [Optimizing delete filesystem confirmation of curvefs-tool.](https://github.com/opencurve/curve/issues/843)
-- [Optimizing leaky bucket algorithm of quality of service.](https://github.com/opencurve/curve/pull/1045)
-
-
-## bug fix
-
-- [Fix insert inode fail when load deleting partition.](https://github.com/opencurve/curve/pull/997)
-- [Fix read fail when write cache of diskcache is enabled, because metadata maybe newer than data in S3.](https://github.com/opencurve/curve/pull/1006)
-- [Fix CurveFS mds heartbeat doesn't delete copyset creating.](https://github.com/opencurve/curve/pull/1011)
-- [Fix CurveFS create copyset when exist already.](https://github.com/opencurve/curve/pull/1002)

CHANGELOG-2.1.md

@@ -1,41 +0,0 @@
-# CHANGELOG of v2.1
-
-## new features
-
-- [CurveFS: a curve filesystem can be mounted by multi fuse clients.](https://github.com/opencurve/curve/pull/1101)
-
-- [CurveFS: support summary info in dir xattr.](https://github.com/opencurve/curve/pull/1150)
-
-- [CurveFS: support Multiple s3.](https://github.com/opencurve/curve/pull/1132)
-
-## optimization
-
-- [CurveFS client: adapter lru list for disk cache.](https://github.com/opencurve/curve/pull/1088)
-
-- [CurveFS: meta balance.](https://github.com/opencurve/curve/pull/1105)
-
-- [CurveFS metaserver: let s3 compact task store a shared copyset node.](https://github.com/opencurve/curve/pull/1165)
-
-- [CurveFS: add metric for mds topology to update metaserver metric.](https://github.com/opencurve/curve/pull/1177)
-
-## bug fix
-
-- [CurveFS client: fix bug of getleader always fails causes stack overflow.](https://github.com/opencurve/curve/pull/1070)
-
-- [CurveFS tool: fix copyset health check error.](https://github.com/opencurve/curve/pull/1024)
-
-- [CurveFS client: fix client release read data cache core dump.](https://github.com/opencurve/curve/pull/1090)
-
-- [CurveBS nbd: fix misspell in log.](https://github.com/opencurve/curve/pull/1073)
-
-- [CurveFS mds: update copyset condidate.](https://github.com/opencurve/curve/pull/1085)
-
-- [CurveFS client: fix bug of rpc overtime time not backoff.](https://github.com/opencurve/curve/pull/1139)
-
-- [CurveFS client: fix bug when upload failed.](https://github.com/opencurve/curve/pull/1130)
-
-- [CurveFS client: fix the problem that chunkCacheManager has been releaed.](https://github.com/opencurve/curve/pull/1179)
-
-- [CurveFS: fix partition num in topology metric.](https://github.com/opencurve/curve/pull/1187)
-
-- [CurveFS client: fix core dump of DataCache::Flush & fix rpc timeout not backoff.](https://github.com/opencurve/curve/pull/1193)

CHANGELOG-2.2.md

@@ -1,23 +0,0 @@
-# CHANGELOG of v2.2
-
-## Feature
-
-- [curvefs: support curvebs as data backend](https://github.com/opencurve/curve/pull/1207)
-
-- [curvefs/metaserver: now we support rocksdb storage](https://github.com/opencurve/curve/pull/1214)
-
-## Optimization
-
-- [optimize copyset creation](https://github.com/opencurve/curve/pull/1211)
-
-- [curvefs/client: s3 adaptor unit test optimization](https://github.com/opencurve/curve/pull/1227)
-
-- [curvefs: xattr summary info support hard link](https://github.com/opencurve/curve/pull/1185)
-
-## Bugfix
-
-- [curvefs/client: fix waitIntervalSec_ is not init](https://github.com/opencurve/curve/pull/1248)
-
-- [metaserver: fix empty configuration after load snapshot](https://github.com/opencurve/curve/pull/1260)
-
-- [curvefs-mds: fix an uninitialized that spawn infinite threads](https://github.com/opencurve/curve/pull/1272)

CHANGELOG-2.3.md

@@ -1,126 +0,0 @@
-# CHANGELOG of v2.3
-
-Previous change logs can be found at [CHANGELOG-2.2](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.2.md)
-
-## Notable Changes
-
-- [CurveFS : adapt to curveadm to support curvebs as data backend](https://github.com/opencurve/curve/pull/1349)
-
-- [CurveFS monitor: promethus](https://github.com/opencurve/curve/pull/1237)
-
-- [CurveFS client: perf optimize](https://github.com/opencurve/curve/pull/1194)
-
-- [CurveFS metaserver: fixed rocksdb storage memory leak caused by unreleasing iterator and transaction.](https://github.com/opencurve/curve/pull/1388)
-
-## new features
-
-- [CurveFS : adapt to curveadm to support curvebs as data backend](https://github.com/opencurve/curve/pull/1349)
-
-- [CurveFS monitor: promethus](https://github.com/opencurve/curve/pull/1237)
-
-- [CurveFS client: add s3.useVirtualAddressing config, default value: false](https://github.com/opencurve/curve/pull/1253)
-
-## optimization
-
-- [CurveFS metaserver: speed up getting inode by padding inode's s3chunk](https://github.com/opencurve/curve/pull/1344)
-
-- [CurveFS client: perf optimize](https://github.com/opencurve/curve/pull/1194)
-
-## Data Performance
-Hardware: 3 nodes (3*mds, 9*metaserver), each with:
- - Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GHz
- - 256G RAM
- - disk cache: INTEL SSDSC2BB80 800G(iops is about 30000+,bw is about 300MB)
-
-  - performance is as follows:
-
-s3 backend is minio and the cto is disable([what is cto](https://github.com/opencurve/curve/blob/master/docs/cn/CurveFS%E6%94%AF%E6%8C%81%E5%A4%9A%E6%8C%82%E8%BD%BD.pdf)). and as you know, the performance of read may associated with read cache hit.
-
-| minio + diskcache(free)| iops/bandwidth | avg-latency(ms) | clat 99.00th (ms) | clat 99.99th (ms) |
-| :----: | :----: | :----: | :----: | :----: |
-| (numjobs 1) (50G filesize) 4k randwrite | 3539 | 0.281 | 1.5 | 16 |
-| (numjobs 1) (50G filesize) 4k randread | 2785 | 0.357 | 0.9 | 5.8|
-| (numjobs 1) (50G filesize) 512k write | 290 MB/s | 1 | 600 ms | 248|
-| (numjobs 1) (50G filesize) 512k read | 216 MB/s | 4.3 | 275 ms | 7.6 |
-
-
-| minio + diskcache(near full)| iops/bandwidth | avg-latency(ms) | clat 99.00th (ms) | clat 99.99th (ms) |
-| :----: | :----: | :----: | :----: | :----: |
-| (numjobs 1) (50G filesize) 4k randwrite | 2988 | 0.3 | 1.2 | 18 |
-| (numjobs 1) (50G filesize) 4k randread | 1559 | 0.6 | 1.9 | 346|
-| (numjobs 1) (50G filesize) 512k write | 266 MB/s | 0.9| 600 ms | 396|
-| (numjobs 1) (50G filesize) 512k read | 82 MB/s | 86 | 275 ms | 901|
-
-
-| minio + diskcache(full)| iops/bandwidth | avg-latency(ms) | clat 99.00th (ms) | clat 99.99th (ms) |
-| :----: | :----: | :----: | :----: | :----: |
-| (numjobs 1) (20G filesize * 5) 4k randwrite | 2860 | 1.7| 14 | 41 |
-| (numjobs 1) (20G filesize * 5) 4k randread | 76 | 65 | 278 | 725|
-| (numjobs 1) (20G filesize * 5) 512k write | 240 MB/s | 10| 278 | 513|
-| (numjobs 1) (20G filesize * 5) 512k read | 192 MB/s | 12 | 40 | 1955 |
-
-## Metadata Performance
-
-Cluster Topology: 3 nodes (3mds, 6metaserver), each metaserver is deployed on a separate SATA SSD
-
-Configuration: use default configuration
-
-Test tool: mdtest v3.4.0
-
-Test cases:
-- case 1: the directory structure is relatively flat and total 130,000 dirs and files.
-mdtest -z 2 -b 3 -I 10000 -d /mountpoint
-
-- case 2: the directory structure is relatively deep and total 204,700 dirs and files.
-mdtest -z 10 -b 2 -I 100 -d /mountpoint
-
-
-| Case | Dir creation | Dir stat | Dir rename | Dir removal | File creation | File stat | File read | File removal | Tree creation | Tree removal |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| case 1 | 1320 | 5853 | 149 | 670 | 1103 | 5858 | 1669 | 1419 | 851 | 64 |
-| case 2 | 1283 | 5205 | 147 | 924 | 1081 | 5316 | 1634 | 1260 | 1302 | 887 |
-
-You can set configuration item ‘fuseClient.enableMultiMountPointRename’ to false in client.conf if you don't need concurrent renames on multiple mountpoints on the same filesystem. It will improve the performance of metadata.
-
-fuseClient.enableMultiMountPointRename=false
-
-| Case | Dir creation | Dir stat | Dir rename | Dir removal | File creation | File stat | File read | File removal | Tree creation | Tree removal |
-| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| case 1 | 1537 | 7885 | 530 | 611 | 1256 | 7998 | 1861 | 1614 | 1050 | 72 |
-| case 2 | 1471 | 6328 | 509 | 1055 | 1237 | 6874 | 1818 | 1454 | 1489 | 1034 |
-
-## bug fix
-
-- [CurveFS mds: fix an uninitialized that spawn infinite threads](https://github.com/opencurve/curve/pull/1270)
-
-- [CurveFS client: fix missing update extent cache](https://github.com/opencurve/curve/pull/1279)
-
-- [CurveFS: fix a compatible issue which caused by modify FsFileType ](https://github.com/opencurve/curve/pull/1359)
-
-- [CurveFS client: fix bug cachedisk never trim](https://github.com/opencurve/curve/pull/1378)
-
-- [CurveFS client: fix bug of io hang](https://github.com/opencurve/curve/pull/1377)
-
-- [CurveFS metaserver: fixed rocksdb storage memory leak caused by unreleasing iterator and transaction.](https://github.com/opencurve/curve/pull/1388)
-
-- [CurveFS client: Adjust the number of retries to the maximum to avoid an error when the number of retries is reached](https://github.com/opencurve/curve/pull/1410)
-
-- [CurveFS : fix clear copyset creating flag when create copyset success](https://github.com/opencurve/curve/pull/1417)
-
-- [CurveFS mds: fix miss set txid when create partition and fix inodeId type](https://github.com/opencurve/curve/pull/1479)
-
-- [CurveFS client: fix the data iteration error when rpc retry.](https://github.com/opencurve/curve/pull/1474)
-
-- [CurveFS metaserver: skip find dentry when loading from snapshot](https://github.com/opencurve/curve/pull/1460)
-
-- [CurveFS : fix compile error which was caused by LatencyUpdater success](https://github.com/opencurve/curve/pull/1508)
-
-- [CurveFS mds: fix create partition error at parallel case](https://github.com/opencurve/curve/pull/1511)
-
-- [CurveFS metaserver: fix the data iteration error when rpc retry.metaserver: recover s3ChunkInfoRemove field for GetOrModify](https://github.com/opencurve/curve/pull/1404)
-
-- [CurveFS metaserver: fixed s3chunkinfo was padding into inode when it wasnt needed](https://github.com/opencurve/curve/pull/1510)
-
-- [Curve common: fix timer bug](https://github.com/opencurve/curve/pull/1492)
-
-- [CurveFS client: fix statfs problem](https://github.com/opencurve/curve/pull/1620)

CHANGELOG-2.4.md

@@ -1,47 +0,0 @@
-# CHANGELOG of v2.4
-
-Previous change logs can be found at [CHANGELOG-2.3](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.3.md)
-
-## Notable Changes
-
-- [Update aws-sdk-cpp version to 1.9](https://github.com/opencurve/curve/pull/1780)
-
-- [add feature of warmup](https://github.com/opencurve/curve/pull/1793)
-
-
-## new features
-
-- [CurveFS: implement setxattr interface](https://github.com/opencurve/curve/pull/1935)
-
-- [update aws-sdk-cpp: add ip and custom port support](https://github.com/opencurve/curve/pull/1795)
-
-- [CurveFS client: add s3.useVirtualAddressing config, default value: false](https://github.com/opencurve/curve/pull/1253)
-
-- [Update aws-sdk-cpp version to 1.9](https://github.com/opencurve/curve/pull/1780)
-
-- [add feature of warmup](https://github.com/opencurve/curve/pull/1793)
-
-## optimization
-
-- [CurveFS: update attr and extent in single rpc](https://github.com/opencurve/curve/pull/1784)
-
-- [copyset schedule select copyset random](https://github.com/opencurve/curve/pull/1811)
-
-- [CurveFS: only update dirty inode metadata](https://github.com/opencurve/curve/pull/1853)
-
-
-## bug fix
-
-- [CurveFS:  fix RefreshInode do not refresh when inode exist](https://github.com/opencurve/curve/pull/1800)
-
-- [fix misuse Locked & Unlocked calling contract](https://github.com/opencurve/curve/pull/1279)
-
-- [make callback of s3async request async to avoid deadlock](https://github.com/opencurve/curve/pull/1854)
-
-- [CurveFS :fix update nlink error ](https://github.com/opencurve/curve/pull/1901)
-
-- [CurveFS client: fix umount bug](https://github.com/opencurve/curve/pull/1923)
-
-- [CurveFS client:  fix refresh inode will overwrite data in cache when enabel cto.](https://github.com/opencurve/curve/pull/1993)
-
-

CHANGELOG-2.5.md

@@ -1,49 +0,0 @@
-# CHANGELOG of v2.5
-
-Previous change logs can be found at [CHANGELOG-2.4](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.4.md)
-
-
-
-## New Features
-
-- Memcache support
-    - [curvefs/client: support memcached cluster](https://github.com/opencurve/curve/pull/2096) @ilixiaocui
-    - [curvefs/mds: support memcache cluster](https://github.com/opencurve/curve/pull/2108) @Cyber-SiKu
-    - [curvefs/client: add global cache client like memcached](https://github.com/opencurve/curve/pull/2102) @fansehep
-    - [curvefs/client: fix client core dump error](https://github.com/opencurve/curve/pull/2157) @ilixiaocui
-    - [add ut](https://github.com/opencurve/curve/pull/2164) @Cyber-SiKu
-
-- curvefs new tools 
-    - [add delete impl](https://github.com/opencurve/curve/pull/2088) @shentupenghui
-    - [feat: [tools-v2] bs list dir](https://github.com/opencurve/curve/pull/2082) @Sindweller
-    - [fix missing docs links](https://github.com/opencurve/curve/pull/2082) @zyb521
-    - [curve/toos-v2: add list client #2037](https://github.com/opencurve/curve/pull/2076) @tsonglew
-    - [support create volume directory by curve_ops_tool](https://github.com/opencurve/curve/pull/2078) @aspirer
-
-- [install: added playground.](https://github.com/opencurve/curve/pull/2053) @Wine93
-
-## Optimization
-
-- [update the braft to vesion v1.1.2](https://github.com/opencurve/curve/pull/2091) @tangwz
-
-- [curvefs/client: local cache policy optimization](https://github.com/opencurve/curve/pull/2064) @Tangruilin
-
-
-- [sync by threadpool](https://github.com/opencurve/curve/pull/1912) @fansehep
-
-- Merge block storage and file storage compilation scripts
-    - [Merge block storage and file storage compilation scripts
-](https://github.com/opencurve/curve/pull/2089) @linshiyx
-    - [Add help_msg in Makefile](https://github.com/opencurve/curve/pull/2133) @linshiyx
-    - [fix image.sh](https://github.com/opencurve/curve/pull/2124) @SeanHai
-
-- [fix bs list zone typo](https://github.com/opencurve/curve/pull/2066) @tsonglew 
-
-- [spell optimize](https://github.com/opencurve/curve/pull/2059) @shentupenghui
-
-
-## Bug Fix
-
-- aws s3 sdk revert
-    - [revert new s3 sdk due to critical bug](https://github.com/opencurve/curve/pull/2149) @h0hmj
-    - [curvefs/client: change the default value of s3](https://github.com/opencurve/curve/pull/2158) @wuhongsong

CODE_OF_CONDUCT.md

@@ -1,76 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
- advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
- address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
- professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at storage_mgm@163.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq

LICENSE

@@ -1,202 +1,152 @@
+Files: *
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+Files: curve-snapshotcloneserver-nginx/docker/Dockerfile
+Copyright: Copyright (c) 2017-2020, Evan Wies evan@neomantra.net.
+License:
+================================================================================
+docker-openresty is licensed under the 2-clause BSD license.
 
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+Copyright (c) 2017-2020, Evan Wies evan@neomantra.net.
 
-   1. Definitions.
+This module is licensed under the terms of the BSD license.
 
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+================================================================================
 
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
+Files: curve-snapshotcloneserver-nginx/app/etc/config.lua
+       curve-snapshotcloneserver-nginx/app/lib/resty/argutils.lua
+       curve-snapshotcloneserver-nginx/app/lib/resty/checkups.lua
+       curve-snapshotcloneserver-nginx/app/lib/resty/checkups/*
+       curve-snapshotcloneserver-nginx/app/src/modules/httpipe.lua
+       curve-snapshotcloneserver-nginx/app/src/modules/httproxy.lua
+       curve-snapshotcloneserver-nginx/app/src/modules/reqlimit.lua
+Copyright: Copyright (C) 2014-2016 UPYUN, Inc.
+License:
+================================================================================
+The bundle itself is licensed under the 2-clause BSD license.
 
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
+Copyright (c) 2016, UPYUN(又拍云) Inc.
 
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
+This module is licensed under the terms of the BSD license.
 
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+================================================================================
 
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
+Files: curve-snapshotcloneserver-nginx/app/lib/resty/uuid.lua
+Copyright: Copyright (c) 2016-2019 Thibault Charbonnier
+License:
+================================================================================
+The MIT License (MIT)
 
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
+    Copyright (c) 2016-2019 Thibault Charbonnier
 
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
 
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
 
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+================================================================================
 
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
+Files: monitor/*
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: GPL 2.0 (see LICENSE_GPL)
 
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
+Files: src/chunkserver/raftsnapshot/curve_file_service.cpp
+       src/chunkserver/raftsnapshot/curve_file_service.h
+       src/chunkserver/raftsnapshot/curve_snapshot.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot.h
+       src/chunkserver/raftsnapshot/curve_snapshot_copier.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_copier.h
+       src/chunkserver/raftsnapshot/curve_snapshot_file_reader.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_file_reader.h
+       src/chunkserver/raftsnapshot/curve_snapshot_reader.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_reader.h
+       src/chunkserver/raftsnapshot/curve_snapshot_storage.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_storage.h
+       src/chunkserver/raftsnapshot/curve_snapshot_writer.cpp
+       src/chunkserver/raftsnapshot/curve_snapshot_writer.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015 Baidu.com, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
+Files: src/common/authenticator.cpp
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2016 Baidu, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
+Files: src/common/hash.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015, Baidu.com
+           Copyright (c) 2011 The LevelDB Authors
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
+Files: src/common/string_util.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015, Baidu.com
+License: Apache 2.0 (see LICENSE_APACHE)
 
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
+Files: src/mds/nameserver2/file_lock.cpp
+       src/mds/nameserver2/file_lock.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2016, Baidu.com, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
+Files: src/tools/nbd/*
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: GPL 2.0 (see LICENSE_GPL)
 
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
+Files: src/tools/nbd/nbd-netlink.h
+Copyright: Copyright (C) 2017 Facebook
+License: GPL 2.0 (see LICENSE_GPL)
 
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
+Files: src/tools/nbd/define.h
+       src/tools/nbd/NBDController.cpp
+       src/tools/nbd/NBDController.h
+       src/tools/nbd/NBDServer.cpp
+       src/tools/nbd/NBDServer.h
+       src/tools/nbd/util.cpp
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (C) 2015 - 2016 Kylin Corporation
+License: LGPL 2.1 (see LICENSE_LGPL)
 
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
+Files: src/tools/nbd/texttable.cpp
+       src/tools/nbd/texttable.h
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (C) 2012 Inktank Storage, Inc.
+License: LGPL 2.1 (see LICENSE_LGPL)
 
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
+Files: test/chunkserver/raftsnapshot/curve_snapshot_storage_test.cpp
+Copyright: Copyright (c) 2020, NetEase Inc.
+           Copyright (c) 2015 Baidu.com, Inc.
+License: Apache 2.0 (see LICENSE_APACHE)
 
-   END OF TERMS AND CONDITIONS
+Files: test/tools/nbd/*
+Copyright: Copyright (c) 2020, NetEase Inc.
+License: GPL 2.0 (see LICENSE_GPL)
 
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+Files: thirdparties/aws/*
+       common.bzl
+Copyright: Copyright 2019 The TensorFlow Authors
+License: Apache 2.0 (see LICENSE_APACHE)

LICENSE_APACHE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

LICENSE_GPL

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

LICENSE_LGPL

@@ -0,0 +1,502 @@
+          GNU LESSER GENERAL PUBLIC LICENSE
+               Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+                Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+          GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+             END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!

MAINTAINERS.md

@@ -1,7 +0,0 @@
-# Maintainers
-
-The current maintainers of the Curve project are:
-
-* Pan WANG, [aspirer](https://github.com/aspirer), <aspirer2004@gmail.com>, core maintainer
-* XiaoCui Li, [ilixiaocui](https://github.com/ilixiaocui), <ilixiaocui@163.com>, core maintainer
-* opencurveadmin, [opencurveadmin](https://github.com/opencurveadmin), <hzchenwei7@corp.netease.com>, project management

Makefile

@@ -1,81 +1,24 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image playground check test
+.PHONY: list build install image clean
 
-stor?=""
 prefix?= "$(PWD)/projects"
 release?= 0
-dep?= 0
 only?= "*"
 tag?= "curvebs:unknown"
-case?= "*"
-os?= "debian9"
-ci?=0
-
-define help_msg
-## list
-Usage:
-    make list stor=bs/fs
-Examples:
-    make list stor=bs
-
-## build
-Usage:
-    make build stor=bs/fs only=TARGET dep=0/1 release=0/1 os=OS
-Examples:
-    make build stor=bs only=//src/chunkserver:chunkserver
-    make build stor=bs only=src/* dep=0
-    make build stor=fs only=test/* os=debian9
-    make build stor=fs release=1
-
-
-## dep
-Usage:
-    make dep stor=bs/fs
-Examples:
-    make dep stor=bs
-
-
-## install
-Usage:
-    make install stor=bs/fs prefix=PREFIX only=TARGET
-Examples:
-    make install stor=bs prefix=/usr/local/curvebs only=*
-    make install stor=bs prefix=/usr/local/curvebs only=chunkserver
-    make install stor=fs prefix=/usr/local/curvefs only=etcd
-
-
-## image
-Usage:
-    make image stor=bs/fs tag=TAG os=OS
-Examples:
-    make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian9
-endef
-export help_msg
-
-help:
-	@echo "$$help_msg"
+tgt_pkg?= ""
 
 list:
-	@bash util/build.sh --stor=$(stor) --list
+	@bash util/build.sh --list
 
 build:
-	@bash util/build.sh --stor=${stor} --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os)
-
-dep:
-	@bash util/build.sh --stor=$(stor) --only="" --dep=1
+	@bash util/build.sh --only=$(only) --release=$(release)
 
 install:
-	@bash util/install.sh --stor=$(stor) --prefix=$(prefix) --only=$(only)
+	@bash util/install.sh --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(stor) $(tag) $(os)
+	@bash util/image.sh $(tag) $(tgt_pkg)
 
-playground:
-	@bash util/playground.sh
-
-check:
-	@bash util/check.sh $(stor)
-
-test:
-	@bash util/test.sh $(stor) $(only)
+clean:
+	@bazel clean

README.md

@@ -1,271 +1,91 @@
+[English version](README_en.md)
 
 
-<div align=center> <img src="docs/images/curve-logo1-nobg.png" width = 45%>
-
-<div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
-
-**A cloud-native distributed storage system**
-
-**A sandbox project hosted by the CNCF Foundation**
-
-#### English | [简体中文](README_cn.md)
-### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
-<div align=left>
-
-<div class="column" align="middle">
-  </a>
-  <a href="https://github.com/opencurve/curve/blob/master/LICENSE">
-    <img src=https://img.shields.io/aur/license/android-studio?style=plastic alt="license"/>
-  </a>
-  <a href="https://github.com/opencurve/curve/releases">
-   <img src="https://img.shields.io/github/v/release/opencurve/curve?include_prereleases" alt="release"/>
-  </a>
-  <a href="https://bestpractices.coreinfrastructure.org/projects/6136">
-    <img src="https://bestpractices.coreinfrastructure.org/projects/6136/badge">
-  </a>
-  <a href="https://github.com/opencurve/curve/tree/master/docs">
-    <img src="https://img.shields.io/badge/docs-latest-green.svg">
-</div>
-
-✨ Contents
-========
-
-  - [About Curve](#about-curve)
-  - [Curve Architecture](#curve-architecture)
-  - [Design Documentation](#design-documentation)
-  - [CurveBS quick start](#curvebs-quick-start)
-    - [Deploy an All-in-one experience environment](#deploy-an-all-in-one-experience-environment)
-    - [FIO curve block storage engine](#fio-curve-block-storage-engine)
-  - [CurveFS quick start](#curvefs-quick-start)
-  - [Test environment configuration](#test-environment-configuration)
-  - [Governance](#governance)
-  - [Contribute us](#contribute-us)
-  - [Code of Conduct](#code-of-conduct)
-  - [LICENSE](#license)
-  - [Release Cycle](#release-cycle)
-  - [Branch](#branch)
-  - [Contributors](#contributors)
-  - [Star History](#star-history)
-  - [Feedback & Contact](#feedback--contact)
-
-##  About Curve
-
-**Curve** is a modern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
-
-The core application scenarios of CurveBS mainly include:
-- the performance, mixed, capacity cloud disk or persistent volume of virtual machine/container, and remote disks of physical machines
-- high-performance separation of storage and computation architecture: high-performance and low latency architecture based on RDMA+SPDK, supporting the separation deployment structure of various databases such as MySQL and Kafka
-
-
-The core application scenarios of CurveFS mainly include:
-- the cost-effective storage in AI training scene
-- the hot and cold data automation layered storage in big data scenarios
-- the cost-effective shared file storage on the public cloud: It can be used for business scenarios such as AI, big data, file sharing
-- Hybrid storage: Hot data is stored in the local IDC, cold data is stored in public cloud
+<img src="docs/images/curve-logo1.png"/>
 
-<details>
-  <summary><b><font=5>High Performance | More stable | Easy Operation | Cloud Native</b></font></summary>
+# CURVE
 
-- High Performance : CurveBS vs CephBS
+[![Jenkins Coverage](https://img.shields.io/jenkins/coverage/cobertura?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fcurve_untest_job%2F)](http://59.111.93.165:8080/job/curve_untest_job/HTML_20Report/)
+[![Robot failover](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fcurve_failover_testjob%2F&label=failover)](http://59.111.93.165:8080/job/curve_failover_testjob/)
+[![Robot interface](https://img.shields.io/jenkins/tests?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fcurve_robot_job%2F)](http://59.111.93.165:8080/job/curve_robot_job/)
+[![BUILD Status](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fopencurve_multijob%2F)](http://59.111.93.165:8080/job/opencurve_multijob/lastBuild)
+[![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://github.com/opencurve/curve/tree/master/docs)
+[![Releases](https://img.shields.io/github/v/release/opencurve/curve?include_prereleases)](https://github.com/opencurve/curve/releases)
+[![LICENSE](https://img.shields.io/badge/licence-Apache--2.0%2FGPL-blue)](https://github.com/opencurve/curve/blob/master/LICENSE)
 
-  CurveBS: v1.2.0
 
-  CephBS: L/N
-  Performance:
-  CurveBS random read and write performance far exceeds CephBS in the block storage scenario.
+CURVE是网易自主设计研发的高性能、高可用、高可靠分布式存储系统，具有非常良好的扩展性。基于该存储底座可以打造适用于不同应用场景的存储系统，如块存储、对象存储、云原生数据库等。当前我们基于CURVE已经实现了高性能块存储系统，支持快照克隆和恢复 ,支持QEMU虚拟机和物理机NBD设备两种挂载方式, 在网易内部作为高性能云盘使用。
 
-  Environment：3 replicas on a 6-node cluster, each node has 20xSATA SSD, 2xE5-2660 v4 and 256GB memory.
+## 设计文档
 
-  Single Vol：
-  <image src="docs/images/1-nbd-en.png">
+- 通过 [CURVE概述](https://opencurve.github.io/) 可以了解 CURVE 架构
+- CURVE相关文档
+  - [NEBD](docs/cn/nebd.md)
+  - [MDS](docs/cn/mds.md)
+  - [Chunkserver](docs/cn/chunkserver_design.md)
+  - [Snapshotcloneserver](docs/cn/snapshotcloneserver.md)
+  - [CURVE质量体系介绍](docs/cn/quality.md)
+  - [CURVE监控体系介绍](docs/cn/monitor.md)
+  - [Client](docs/cn/curve-client.md)
+  - [Client Python API](docs/cn/curve-client-python-api.md)
+- CURVE上层应用
+  - [对接k8s文档](docs/cn/k8s_csi_interface.md)
 
-  Multi Vols：
-  <image src="docs/images/10-nbd-en.png">
+## 快速开始
 
-- More stable
-  - The stability of the common abnormal Curve is better than that of Ceph in the block storage scenario.
-    | Fault Case | One Disk Failure | Slow Disk Detect | One Server Failure | Server Suspend Animation |
-    | :----: | :----: | :----: | :----: | :----: |
-    | CephBS | jitter 7s | Continuous io jitter | jitter 7s | unrecoverable |
-    | CurveBS | jitter 4s | no effect | jitter 4s | jitter 4s |
-
-- Easy Operation
-  - We have developed [CurveAdm](https://github.com/opencurve/curveadm/wiki) to help O&M staff.
-    | tools  |CephAdm | CurveAdm|
-    | :--: | :--: |:--: |
-    | easy Installation |  ✔️ |   ✔️ |
-    | easy Deployment| ❌(slightly more steps) |  ✔️ |
-    | playground | ❌|   ✔️|
-    | Multi-Cluster Management | ❌ |   ✔️ |
-    | easy Expansion | ❌(slightly more steps)|   ✔️|
-    |easy Upgrade |  ✔️ |   ✔️|
-    |easy to stop service | ❌ |   ✔️|
-    |easy Cleaning | ❌ |   ✔️ |
-    |Deployment environment testing|  ❌ |  ✔️ |
-    |Operational audit|  ❌ |   ✔️|
-    |Peripheral component deployment| ❌ |   ✔️|
-    |easy log reporting| ❌ |   ✔️|
-    |Cluster status statistics reporting| ❌|   ✔️|
-    |Error code classification and solutions|  ❌ |   ✔️|
-  - Ops
-    CurveBS ops is more friendly than CephBS in the block storage scenario.
-    | Ops scenarios | Upgrade clients | Balance |
-    | :----: | :----: | :----: |
-    | CephBS | do not support live upgrade | via plug-in with IO influence |
-    | CurveBS | support live upgrade with second jitter | auto with no influence on IO |
-
-- Cloud Native
-  - Please see [Our understanding of cloud native](https://github.com/opencurve/curve/wiki/Roadmap).
-
-  </details>
-<details>
-  <summary><b><font=5>Docking OpenStack</b></font></summary>
-
-- Please see [Curve-cinder](https://github.com/opencurve/curve-cinder).
+在您开始动手部署前请先仔细阅读特别说明部分：[特别说明](docs/cn/deploy.md#%E7%89%B9%E5%88%AB%E8%AF%B4%E6%98%8E)
 
+### 部署All-in-one体验环境
 
-</details>
-<details>
-  <summary><b><font=5>Docking Kubernetes</b></font></summary>
+[单机部署](docs/cn/deploy.md#%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2)
 
-- Use [Curve CSI Driver](https://github.com/opencurve/curve-csi), The plugin implements the Container Storage Interface(CSI) between Container Orchestrator(CO) and Curve cluster. It allows dynamically provisioning curve volumes and attaching them to workloads.
-- For details of the documentation, see [CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md).
-</details>
-<details>
-  <summary><b><font=5>Docking PolarDB | PG </b></font></summary>
+### 部署多机集群
 
-- It serves as the underlying storage base for [polardb for postgresql](https://github.com/ApsaraDB/PolarDB-for-PostgreSQL) in the form of storage and computation separation, providing data consistency assurance for upper layer database applications, extreme elasticity scaling, and high performance HTAP.
+[多机部署](docs/cn/deploy.md#%E5%A4%9A%E6%9C%BA%E9%83%A8%E7%BD%B2)
 
-- Deployment details can be found at [PolarDB | PG Advanced Deployment(CurveBS)](https://apsaradb.github.io/PolarDB-for-PostgreSQL/zh/deploying/storage-curvebs.html).
+### 查询工具说明
 
+[查询工具说明](docs/cn/curve_ops_tool.md)
 
-</details>
-<details>
-  <summary><b><font=5> More...</b></font></summary>
+## 参与开发
 
-- Curve can also be used as cloud storage middleware using S3-compatible object storage as the data storage engine, providing cost-effective shared file storage for public cloud users.
-</details>
 
-## Curve Architecture
+### 部署编译开发环境
 
-<div align=center> <image src="docs/images/Curve-arch.png" width=60%>
-<div align=left>
+[编译开发环境搭建](docs/cn/build_and_run.md)
 
-<details>
-  <summary><b><font=4>Curve on Hybrid Cloud</b></font></summary>
+### 测试用例编译及运行
+[测试用例编译及运行](docs/cn/build_and_run.md#%E6%B5%8B%E8%AF%95%E7%94%A8%E4%BE%8B%E7%BC%96%E8%AF%91%E5%8F%8A%E6%89%A7%E8%A1%8C)
 
-Curve supports deployment in private and public cloud environments, and can also be used in a hybrid cloud:
-<div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
-<div align=left>
+### 编码规范
+CURVE编码规范严格按照[Google C++开源项目编码指南](https://zh-google-styleguide.readthedocs.io/en/latest/google-cpp-styleguide/contents/)来进行代码编写，请您也遵循这一指南来提交您的代码。
 
-One of them, CurveFS shared file storage system, can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
+### 测试覆盖率要求
+1. 单元测试：增量行覆盖80%以上，增量分支覆盖70%以上
+2. 集成测试：与单元测试合并统计，满足上述覆盖率要求即可
+3. 异常测试：暂不做要求
 
-</details>
+### 其他开发流程说明
+代码开发完成之后，提[pr](https://github.com/opencurve/curve/compare)到curve的master分支。提交pr时，请填写pr模板。pr提交之后会自动触发CI，CI通过并且经过review之后，代码才可合入。
+具体规则请见[CONTRIBUTING](https://github.com/opencurve/curve/blob/master/CONTRIBUTING.md).
 
-<div align=left>
+## 版本发布周期
+- CURVE版本发布周期：大版本半年，小版本1~2个月
+- 版本号规则：采用3段式版本号，x.y.z{-后缀}，x是大版本，y是小版本，z是bugfix，后缀用来区beta版本(-beta)、rc版本(-rc)、和稳定版本(没有后缀)。每半年的大版本是指x增加1，每1~2个月的小版本是y增加1。正式版本发布之后，如果有bugfix是z增加1。
 
-<details>
-  <summary><b><font=4>Curve on Public Cloud</b></font></summary>
+## 分支规则
+所有的开发都在master分支开发，如果需要发布版本，从master拉取新的分支**release-x.y**。版本发布从release-x.y分支发布。
 
-In a public cloud environment, users can deploy CurveFS clusters to replace the shared file storage system provided by cloud vendors and use cloud disks for acceleration, which can greatly reduce business costs, with the following deployment architecture:
-<div align=center>
- <image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
+## 反馈及交流
 
-</details>
-<div align=left>
+- [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询
+- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中
+- User group：当前为微信群，由于群人数过多，需要先添加以下个人微信，再邀请进群。
 
-##  Design Documentation
+<img src="https://raw.githubusercontent.com/opencurve/opencurve.github.io/master/image/curve-wechat.jpeg" style="zoom: 75%;" />
 
-- Wanna have a glance at Curve? Click here for [Intro to Curve](https://www.opencurve.io/)!
-- Want more details about CurveBS? Our documentation for every component:
-  - [NEBD](docs/en/nebd_en.md)
-  - [MDS](docs/en/mds_en.md)
-  - [Chunkserver](docs/en/chunkserver_design_en.md)
-  - [Snapshotcloneserver](docs/en/snapshotcloneserver_en.md)
-  - [CURVE quality control](docs/en/quality_en.md)
-  - [CURVE monitoring](docs/en/monitor_en.md)
-  - [Client](docs/en/client_en.md)
-  - [Client Python API](docs/en/curve-client-python-api_en.md)
-- Application based on CurveBS
-  - [Work with k8s](docs/en/k8s_csi_interface_en.md)
-- CurveFS documentations
-  - [Architecture design](docs/cn/curvefs_architecture.md)
-  - [Client design](docs/cn/curvefs-client-design.md)
-  - [Metadata management](docs/cn/curvefs-metaserver-overview.md)
-  - [Data caching](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%94%AF%E6%8C%81S3%20%E6%95%B0%E6%8D%AE%E7%BC%93%E5%AD%98%E6%96%B9%E6%A1%88.pdf)
-  - [Space allocation](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E7%A9%BA%E9%97%B4%E5%88%86%E9%85%8D%E6%96%B9%E6%A1%88.pdf)
-  - [more details](https://github.com/opencurve/curve-meetup-slides/tree/main/CurveFS)
 
-- CurveAdm
-  - [CurveAdm Doc](https://github.com/opencurve/curveadm/wiki)
 
-##  CurveBS quick start
-In order to improve the operation and maintenance convenience of Curve, we designed and developed the [CurveAdm](https://github.com/opencurve/curveadm) project, which is mainly used for deploying and managing Curve clusters. Currently, it supports the deployment of CurveBS & CurveFS (scaleout, upgrade and other functions are under development), please refer to the [CurveAdm User Manual](https://github.com/opencurve/curveadm/wiki) for related documentation, and install the CurveAdm tool according to the manual before deploying the Curve cluster.
 
 
-###  Deploy an All-in-one experience environment
 
-Please refer to the [CurveBS cluster deployment steps](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment) in the CurveAdm user manual. For standalone experience, please use the "Cluster Topology File - Standalone Deployment" template.
-
-[The command tools' instructions](docs/cn/curve_ops_tool.md)
-### FIO Curve block storage engine
-Fio Curve engine is added, you can clone https://github.com/opencurve/fio and compile the fio tool with our engine(depend on nebd lib), fio command line example:
-```bash
-$ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
-```
-
-If you have any questions during performance testing, please check the [Curve block storage performance tuning guide](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md).
-
-##  CurveFS quick start
-Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
-
-## Test environment configuration
-
-Please refer to the [Test environment configuration](docs/cn/测试环境配置信息.md)
-
-## Practical
-- [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
-- [CurveFS+MinIO S3 Gateway](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
-
-## Governance
-See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
-
-##  Contribute us
-Participation in the Curve project is described in the [Curve Developers Guidelines](developers_guide.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
-We welcome your contribution!
-
-## Code of Conduct
-Curve follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
-
-## LICENSE
-Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
-
-##  Release Cycle
-- CURVE release cycle：Half a year for major version, 1~2 months for minor version
-
-- Versioning format: We use a sequence of three digits and a suffix (x.y.z{-suffix}), x is the major version, y is the minor version, and z is for bugfix. The suffix is for distinguishing beta (-beta), RC (-rc) and GA version (without any suffix). Major version x will increase 1 every half year, and y will increase every 1~2 months. After a version is released, number z will increase if there's any bugfix.
-
-## Branch
-All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
-
-## Contributors
-This project exists thanks to all the people who contribute.
-
-<a href="https://github.com/opencurve/curve/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=opencurve/curve" />
-</a>
-
-Made with [contrib.rocks](https://contrib.rocks).
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=opencurve/curve&type=Date)](https://star-history.com/#opencurve/curve&Date)
-
-## Feedback & Contact
-- [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
-- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.
-- User group：We use Wechat group currently.
-- [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2022): We have an online community meeting every two weeks which talk about what Curve is doing and planning to do. The time and links of the meeting are public in the user group and [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2022).
-
-
-<img src="docs/images/curve-wechat.jpeg" style="zoom: 65%;" />

README_cn.md

@@ -1,256 +0,0 @@
-
-
-<div align=center> <img src="docs/images/curve-logo1-nobg.png" width = 45%>
-
-<div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
-
-**云原生高性能分布式存储系统**
-
-**CNCF基金会的沙箱托管项目**
-
-#### [English](README.md) | 简体中文
-### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
-<div align=left>
-
-<div class="column" align="middle">
-  </a>
-  <a href="https://github.com/opencurve/curve/blob/master/LICENSE">
-    <img src=https://img.shields.io/aur/license/android-studio?style=plastic alt="license"/>
-  </a>
-  <a href="https://github.com/opencurve/curve/releases">
-   <img src="https://img.shields.io/github/v/release/opencurve/curve?include_prereleases" alt="release"/>
-  </a>
-  <a href="https://bestpractices.coreinfrastructure.org/projects/6136">
-    <img src="https://bestpractices.coreinfrastructure.org/projects/6136/badge">
-  </a>
-  <a href="https://github.com/opencurve/curve/tree/master/docs">
-    <img src="https://img.shields.io/badge/docs-latest-green.svg">
-</div>
-
-✨ 目录
-========
-
-  - [关于 Curve](#关于-curve)
-  - [Curve 架构](#curve-架构)
-  - [设计文档](#设计文档)
-  - [CurveBS 快速体验](#curvebs-快速体验)
-    - [部署All-in-one体验环境](#部署all-in-one体验环境)
-    - [FIO curve块存储引擎](#fio-curve块存储引擎)
-  - [CurveFS 快速体验](#curvefs-快速体验)
-  - [测试环境配置](#测试环境配置)
-  - [社区治理](#社区治理)
-  - [贡献我们](#贡献我们)
-  - [行为守则](#行为守则)
-  - [LICENSE](#license)
-  - [版本发布周期](#版本发布周期)
-  - [分支规则](#分支规则)
-  - [反馈及交流](#反馈及交流)
-
-
-##  关于 Curve
-**Curve** 是网易主导自研的现代化存储系统, 目前支持文件存储(CurveFS)和块存储(CurveBS)。现作为沙箱项目托管于[CNCF](https://www.cncf.io/)。
-
-CurveBS的核心应用场景主要包括：
-- 虚拟机/容器的性能型、混合型、容量型云盘或持久化卷，以及物理机的远程存储盘
-- 高性能存算分离架构：基于RDMA+SPDK的高性能低时延架构，支撑MySQL、kafka等各类数据库、中间件的存算分离部署架构，提升实例交付效率和资源利用率
-
-
-CurveFS的核心应用场景主要包括：
-- AI训练（含机器学习等）场景下的高性价比存储
-- 大数据场景下的冷热数据自动化分层存储
-- 公有云上高性价比的共享文件存储：可用于AI、大数据、文件共享等业务场景
-- 混合云存储：热数据存储在本地IDC，冷数据存储在公有云
-
-
-<details>
-  <summary><b><font=5>高性能 | 更稳定 | 易运维 | 云原生</b></font></summary>
-
-- 高性能 : CurveBS vs CephBS
-
-  CurveBS: v1.2.0
-
-  CephBS: L/N
-  性能:
-  块存储场景下，CurveBS随机读写性能远优于CephBS。
-  测试环境：6台服务器*20块SATA SSD，E5-2660 v4，256G，3副本，使用nbd场景。
-
-  单卷场景：
-  <image src="docs/images/1-nbd.jpg">
-
-  多卷场景：
-  <image src="docs/images/10-nbd.jpg">
-
-- 更稳定
-  - 块存储场景下，常见异常CurveBS的稳定性优于CephBS。
-
-    | 异常场景 | 单盘故障 | 慢盘 | 机器宕机 | 机器卡住 |
-    | :----: | :----: | :----: | :----: | :----: |
-    | CephBS | 抖动7s | 持续io抖动 | 抖动7s | 不可恢复 |
-    | CurveBS | 抖动4s | 无影响 | 抖动4s | 抖动4s |
-
-- 易运维
-  - 我们开发了 [CurveAdm](https://github.com/opencurve/curveadm/wiki)来帮助运维人员。
-
-    | 工具 |CephAdm | CurveAdm|
-    | :--:  | :--: |:--: |
-    | 一键安装  |   ✔️ |   ✔️ |
-    | 一键部署  | ❌(步骤稍多) |  ✔️ |
-    | playground |  ❌|   ✔️|
-    | 多集群管理  |  ❌ |   ✔️ |
-    | 一键扩容 | ❌(步骤稍多)|   ✔️|
-    |一键升级 |   ✔️ |   ✔️|
-    |一键停服 |  ❌ |   ✔️|
-    |一键清理 |  ❌ |   ✔️ |
-    |部署环境检测|  ❌ |   ✔️ |
-    |操作审计| ❌ |   ✔️|
-    |周边组件部署| ❌ |   ✔️|
-    |一键日志上报| ❌ |   ✔️|
-    |集群状态统计上报|  ❌|   ✔️|
-    |错误码分类及解决方案|  ❌ |   ✔️|
-  - 运维
-    块存储场景下，CurveBS常见运维更友好。
-
-    | 运维场景 | 客户端升级 | 均衡 |
-    | :----: | :----: | :----: |
-    | CephBS | 不支持热升级 | 外部插件调整，影响业务IO |
-    | CurveBS | 支持热升级，秒级抖动 | 自动均衡，对业务IO无影响 |
-- 云原生
-  - 详见[我们对云原生的理解](https://github.com/opencurve/curve/wiki/Roadmap_CN)。
-
-  </details>
-<details>
-  <summary><b><font=5>对接 OpenStack</b></font></summary>
-
-- 详见 [Curve-cinder](https://github.com/opencurve/curve-cinder)。
-
-
-</details>
-<details>
-  <summary><b><font=5>对接 Kubernetes</b></font></summary>
-
-- 使用 [Curve CSI Driver](https://github.com/opencurve/curve-csi) 插件在 Container Orchestrator (CO) 与 Curve 集群中实现了 Container Storage Interface(CSI)。
-- 文档详见[CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md)。
-</details>
-<details>
-  <summary><b><font=5>对接 PolarDB | PG </b></font></summary>
-
-- 作为存算分离形态分布式数据库 [PolarDB | PG](https://github.com/ApsaraDB/PolarDB-for-PostgreSQL) 底层存储底座, 为上层数据库应用提供数据一致性保证, 极致弹性, 高性能HTAP。部署详见[PolarDB | PG 进阶部署(CurveBS)](https://apsaradb.github.io/PolarDB-for-PostgreSQL/zh/deploying/storage-curvebs.html)。
-
-
-</details>
-<details>
-  <summary><b><font=5> 更多...</b></font></summary>
-
-- Curve 亦可作为云存储中间件使用 S3 兼容的对象存储作为数据存储引擎，为公有云用户提供高性价比的共享文件存储。
-</details>
-
-## Curve 架构
-
-<div align=center> <image src="docs/images/Curve-arch.png" width=60%>
-<div align=left>
-
-<details>
-  <summary><b><font=4>Curve混合云支持</b></font></summary>
-
-Curve支持部署在私有云和公有云环境，也可以以混合云方式使用，私有云环境下的部署架构如下：
-<div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
-<div align=left>
-
-其中CurveFS共享文件存储系统可以弹性伸缩到公有云存储，可以为用户提供更大的容量弹性、更低的成本、更好的性能体验。
-
-</details>
-
-<div align=left>
-
-<details>
-  <summary><b><font=4>Curve公有云支持</b></font></summary>
-
-公有云环境下，用户可以部署CurveFS集群，用来替换云厂商提供的共享文件存储系统，并利用云盘进行加速，可极大的降低业务成本，其部署架构如下：
-<div align=center>
- <image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
-
-</details>
-<div align=left>
-
-
-## 设计文档
-
-- 通过 [Curve概述](https://opencurve.github.io/) 可以了解 Curve 架构。
-- CurveBS相关文档
-  - [NEBD](docs/cn/nebd.md)
-  - [MDS](docs/cn/mds.md)
-  - [Chunkserver](docs/cn/chunkserver_design.md)
-  - [Snapshotcloneserver](docs/cn/snapshotcloneserver.md)
-  - [Curve质量体系介绍](docs/cn/quality.md)
-  - [Curve监控体系介绍](docs/cn/monitor.md)
-  - [Client](docs/cn/curve-client.md)
-  - [Client Python API](docs/cn/curve-client-python-api.md)
-- CurveBS上层应用
-  - [对接k8s文档](docs/cn/k8s_csi_interface.md)
-- CurveFS相关文档
-  - [架构设计](docs/cn/curvefs_architecture.md)
-  - [Client概要设计](docs/cn/curvefs-client-design.md)
-  - [元数据管理](docs/cn/curvefs-metaserver-overview.md)
-  - [数据缓存方案](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%94%AF%E6%8C%81S3%20%E6%95%B0%E6%8D%AE%E7%BC%93%E5%AD%98%E6%96%B9%E6%A1%88.pdf)
-  - [空间分配方案](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E7%A9%BA%E9%97%B4%E5%88%86%E9%85%8D%E6%96%B9%E6%A1%88.pdf)
-  - [更多](https://github.com/opencurve/curve-meetup-slides/tree/main/CurveFS)
-- CurveAdm相关文档
-  - [Wiki](https://github.com/opencurve/curveadm/wiki)
-
-##  CurveBS 快速体验
-为了提升 Curve 的运维便利性，我们设计开发了 [CurveAdm](https://github.com/opencurve/curveadm) 项目，其主要用于部署和管理 Curve 集群，目前已支持部署CurveBS & CurveFS（扩容、版本升级等更多功能正在开发中），相关使用文档请参考 [CurveAdm用户手册](https://github.com/opencurve/curveadm/wiki)，并根据手册首先安装CurveAdm工具之后再进行Curve集群的部署。
-
-###  部署All-in-one体验环境
-请参考CurveAdm用户手册中[CurveBS集群部署步骤](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment)，单机体验环境请使用“集群拓扑文件-单机部署”模板。
-
-curve 提供了命令行工具以查看集群状态和进行基本集群操作:[命令行工具说明](docs/cn/curve_ops_tool.md)
-### FIO Curve块存储引擎
-fio的Curve块存储引擎代码已经上传到 https://github.com/opencurve/fio ，请自行编译测试（依赖nebd库），fio命令行示例：
-```bash
-$ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
-```
-
-在性能测试过程中有任何问题，请查看[Curve块存储性能调优指南](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md)
-
-##  CurveFS 快速体验
-请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
-
-## 测试环境配置
-
-请参考 [测试环境配置](docs/cn/测试环境配置信息.md)
-##  社区治理
-请参考[社区治理](https://github.com/opencurve/community/blob/master/GOVERNANCE.md)。
-
-##  贡献我们
-
-参与 Curve 项目开发详见[Curve 开发者指南](developers_guide_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
-
-## 最佳实践
-- [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
-- [CurveFS+S3网关部署实践](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
-
-##  行为守则
-Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。
-
-## LICENSE
-Curve 在 [Apache 2.0](LICENSE) 协议下进行分发。
-
-##  版本发布周期
-- CURVE版本发布周期：大版本半年，小版本1~2个月
-- 版本号规则：采用3段式版本号，x.y.z{-后缀}，x是大版本，y是小版本，z是bugfix，后缀用来区beta版本(-beta)、rc版本(-rc)、和稳定版本(没有后缀)。每半年的大版本是指x增加1，每1~2个月的小版本是y增加1。正式版本发布之后，如果有bugfix是z增加1。
-
-##  分支规则
-所有的开发都在master分支开发，如果需要发布版本，从master拉取新的分支**release-x.y**。版本发布从release-x.y分支发布。
-
-##  反馈及交流
-
-- [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询。
-- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中。
-- User group：当前为微信群，由于群人数过多，需要先添加以下个人微信，再邀请进群。
-
-<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />
-
-
-
-
-

README_en.md

@@ -0,0 +1,89 @@
+[中文版](README.md)
+
+
+<img src="docs/images/curve-logo1.png"/>
+
+# CURVE
+
+[![Jenkins Coverage](https://img.shields.io/jenkins/coverage/cobertura?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fcurve_untest_job%2F)](http://59.111.93.165:8080/job/curve_untest_job/HTML_20Report/)
+[![Robot failover](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fcurve_failover_testjob%2F&label=failover)](http://59.111.93.165:8080/job/curve_failover_testjob/)
+[![Robot interface](https://img.shields.io/jenkins/tests?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fcurve_robot_job%2F)](http://59.111.93.165:8080/job/curve_robot_job/)
+[![BUILD Status](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.93.165%3A8080%2Fjob%2Fopencurve_multijob%2F)](http://59.111.93.165:8080/job/opencurve_multijob/lastBuild)
+[![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://github.com/opencurve/curve/tree/master/docs)
+[![Releases](https://img.shields.io/github/v/release/opencurve/curve?include_prereleases)](https://github.com/opencurve/curve/releases)
+[![LICENSE](https://img.shields.io/badge/licence-Apache--2.0%2FGPL-blue)](https://github.com/opencurve/curve/blob/master/LICENSE)
+
+CURVE is a distributed storage system designed and developed independently by NetEase, featured with high performance, high availability, high reliability and well expansibility, and it can serve as the basis for storage systems designed for different scenario (e.g. block storage, object storage and cloud database). 
+
+So far, we have implemented a high performance block storage system, which supports snapshot, clone and recovery, and it can be attached on QEMU virtual machine or physical machine (by curve-nbd). CURVE has been served as an elastic block storage service inside NetEase for a certain time, during which high performance and reliability have shown.
+
+## Design Documentation
+
+- Wanna have a glance at CURVE? Click here for [Intro to CURVE](https://opencurve.github.io/)!
+- Want more details? Our documentation for every component:
+  - [NEBD](docs/en/nebd_en.md)
+  - [MDS](docs/en/mds_en.md)
+  - [Chunkserver](docs/cn/chunkserver_design_en.md)
+  - [Snapshotcloneserver](docs/cn/snapshotcloneserver_en.md)
+  - [CURVE quality control](docs/cn/quality.md)
+  - [CURVE monitoring](docs/cn/monitor.md)
+  - [Client](docs/cn/curve-client.md)
+  - [Client Python API](docs/cn/curve-client-python-api.md)
+- Application based on CURVE
+  - [Work with k8s](docs/cn/k8s_csi_interface.md)
+
+## Quick Start
+
+Want to try on it? Take it easy! We'll help you step by step, but make sure you've read this [Special Statement](docs/cn/deploy.md#%E7%89%B9%E5%88%AB%E8%AF%B4%E6%98%8E) before you start.
+
+### Deploy an all-in-one environment (to try how CURVE works)
+
+[Deploy on single machine](docs/cn/deploy.md#%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2)
+
+### Deploy multi-machine cluster (try it in production environment)
+
+[Deploy on multiple machines](docs/cn/deploy.md#%E5%A4%9A%E6%9C%BA%E9%83%A8%E7%BD%B2)
+
+### curve_ops_tool introduction
+
+[curve_ops_tool introduction](docs/cn/curve_ops_tool.md)
+
+## For Developers
+
+### Deploy build and development environment
+
+[development environment deployment](docs/cn/build_and_run.md)
+
+### Compile test cases and run
+[test cases compiling and running](docs/cn/build_and_run.md#%E6%B5%8B%E8%AF%95%E7%94%A8%E4%BE%8B%E7%BC%96%E8%AF%91%E5%8F%8A%E6%89%A7%E8%A1%8C)
+
+### Coding style guides
+CURVE is coded following [Google C++ Style Guide strictly](https://google.github.io/styleguide/cppguide.html). Please follow this guideline if you're trying to contribute your codes.
+
+### Code coverage requirement
+1. Unit tests: Incremental line coverage ≥ 80%, incremental branch coverage ≥ 70%
+2. Integration tests: Measure together with unit tests, and should fulfill the same requirement
+3. Exception tests: Not required yet
+
+### Other processes
+
+After finishing the development of your code, you should submit a pull request to master branch of CURVE and fill out a pull request template. The pull request will trigger the CI automatically, and the code will only be merged after passing the CI and being reviewed.
+
+For more detail, please refer to [CONTRIBUTING](https://github.com/opencurve/curve/blob/master/CONTRIBUTING.md).
+
+## Release Cycle
+- CURVE release cycle：Half a year for major version, 1~2 months for minor version
+
+- Versioning format: We use a sequence of three digits and a suffix (x.y.z{-suffix}), x is the major version, y is the minor version, and z is for bugfix. The suffix is for distinguishing beta (-beta), RC (-rc) and GA version (without any suffix). Major version x will increase 1 every half year, and y will increase every 1~2 months. After a version is released, number z will increase if there's any bugfix.
+
+## Branch
+
+All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
+
+## Feedback & Contact
+
+- [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
+- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.
+- User group：We use Wechat group currently.
+
+<img src="https://raw.githubusercontent.com/opencurve/opencurve.github.io/master/image/curve-wechat.jpeg" style="zoom: 75%;" />

WORKSPACE

@@ -16,25 +16,12 @@
 
 workspace(name = "curve")
 
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-# skylib
-http_archive(
-    name = "bazel_skylib",
-    urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.0/bazel-skylib-1.2.0.tar.gz",
-    ],
-    sha256 = "af87959afe497dc8dfd4c6cb66e1279cb98ccc84284619ebfec27d9c09a903de",
-)
-load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
-bazel_skylib_workspace()
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 git_repository(
     name = "com_github_baidu_braft",
     remote = "https://github.com/baidu/braft",
-    commit = "d12de388c97998f5ccd5cb97ed0da728815ef438",
+    commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e",
 )
 
 bind(
@@ -45,28 +32,11 @@ bind(
 # proto_library, cc_proto_library, and java_proto_library rules implicitly
 # depend on @com_google_protobuf for protoc and proto runtimes.
 # This statement defines the @com_google_protobuf repo.
-
-# zlib
-http_archive(
-    name = "net_zlib",
-    build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
-    strip_prefix = "zlib-1.2.11",
-    urls = ["https://zlib.net/zlib-1.2.11.tar.gz"],
-)
-
-bind(
-    name = "zlib",
-    actual = "@net_zlib//:zlib",
-)
-
 http_archive(
     name = "com_google_protobuf",
-    strip_prefix = "protobuf-3.6.1.3",
-    patch_args = ["-p1"],
-    patches = ["//:thirdparties/protobuf/protobuf.patch"],
-    sha256 = "9510dd2afc29e7245e9e884336f848c8a6600a14ae726adb6befdb4f786f0be2",
-    urls = ["https://github.com/google/protobuf/archive/v3.6.1.3.zip"],
+    sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30",
+    strip_prefix = "protobuf-3.5.0",
+    urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"],
 )
 
 bind(
@@ -75,14 +45,11 @@ bind(
 )
 
 #import the  gtest files.
-http_archive(
+new_git_repository(
     name = "com_google_googletest",
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
-        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
-    ],
-    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
-    strip_prefix = "googletest-release-1.12.1",
+    build_file = "bazel/gmock.BUILD",
+    remote = "https://github.com/google/googletest",
+    tag = "release-1.8.0",
 )
 
 bind(
@@ -109,7 +76,10 @@ bind(
 http_archive(
     name = "com_github_gflags_gflags",
     strip_prefix = "gflags-2.2.2",
-    urls = ["https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"],
+    urls = [
+        "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz",
+        "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
+    ],
 )
 
 bind(
@@ -117,11 +87,11 @@ bind(
     actual = "@com_github_gflags_gflags//:gflags",
 )
 
-http_archive(
+new_http_archive(
     name = "com_github_google_leveldb",
-    build_file = "@com_github_brpc_brpc//:leveldb.BUILD",
+    build_file = "bazel/leveldb.BUILD",
     strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
-    urls = ["https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"],
+    url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
 )
 
 bind(
@@ -130,7 +100,7 @@ bind(
 )
 
 git_repository(
-    name = "com_github_brpc_brpc",
+    name = "com_github_apache_brpc",
     remote = "https://github.com/apache/incubator-brpc",
     commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
     patches = ["//:thirdparties/brpc/brpc.patch"],
@@ -139,28 +109,28 @@ git_repository(
 
 bind(
     name = "brpc",
-    actual = "@com_github_brpc_brpc//:brpc",
+    actual = "@com_github_apache_brpc//:brpc",
 )
 
 bind(
     name = "butil",
-    actual = "@com_github_brpc_brpc//:butil",
+    actual = "@com_github_apache_brpc//:butil",
 )
 
 bind(
     name = "bthread",
-    actual = "@com_github_brpc_brpc//:bthread",
+    actual = "@com_github_apache_brpc//:bthread",
 )
 
 bind(
     name = "bvar",
-    actual = "@com_github_brpc_brpc//:bvar",
+    actual = "@com_github_apache_brpc//:bvar",
 )
 
 # jsoncpp
 new_git_repository(
     name = "jsoncpp",
-    build_file = "//:thirdparties/jsoncpp.BUILD",
+    build_file = "bazel/jsoncpp.BUILD",
     remote = "https://github.com/open-source-parsers/jsoncpp.git",
     tag = "1.8.4",
 )
@@ -172,44 +142,49 @@ bind(
 
 new_local_repository(
     name = "etcdclient",
-    build_file = "//:thirdparties/etcdclient.BUILD",
+    build_file = "bazel/etcdclient.BUILD",
     path = "thirdparties/etcdclient",
 )
 
-new_local_repository(
-    name = "libmemcached",
-    build_file = "//:thirdparties/memcache/memcache.BUILD",
-    path = "thirdparties/memcache/libmemcached-1.1.2",
-)
-
-
-http_archive(
+new_http_archive(
     name = "aws",
-    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
+    urls = [
+        "https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
+        "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz",
+    ],
     sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
     strip_prefix = "aws-sdk-cpp-1.7.340",
     build_file = "//:thirdparties/aws/aws.BUILD",
 )
 
-http_archive(
+new_http_archive(
     name = "aws_c_common",
-    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
+    urls = [
+        "https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
+        "https://mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
+    ],
     sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
     strip_prefix = "aws-c-common-0.4.29",
     build_file = "//:thirdparties/aws/aws-c-common.BUILD",
 )
 
-http_archive(
+new_http_archive(
     name = "aws_c_event_stream",
-    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
+    urls = [
+        "https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
+        "https://mirror.tensorflow.org/github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz",
+    ],
     sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
     strip_prefix = "aws-c-event-stream-0.1.4",
     build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
 )
 
-http_archive(
+new_http_archive(
     name = "aws_checksums",
-    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
+    urls = [
+        "https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
+        "https://mirror.tensorflow.org/github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz",
+    ],
     sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
     strip_prefix = "aws-checksums-0.1.5",
     build_file = "//:thirdparties/aws/aws-checksums.BUILD",
@@ -230,53 +205,3 @@ http_archive(
   strip_prefix = "abseil-cpp-20210324.2",
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
-
-# fmt
-http_archive(
-  name = "fmt",
-  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
-  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
-  strip_prefix = "fmt-9.1.0",
-  build_file = "//:thirdparties/fmt.BUILD",
-)
-
-# spdlog
-http_archive(
-  name = "spdlog",
-  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
-  strip_prefix = "spdlog-1.11.0",
-  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
-  build_file = "//:thirdparties/spdlog.BUILD",
-)
-
-# Bazel platform rules.
-http_archive(
-    name = "platforms",
-    sha256 = "b601beaf841244de5c5a50d2b2eddd34839788000fa1be4260ce6603ca0d8eb7",
-    strip_prefix = "platforms-98939346da932eef0b54cf808622f5bb0928f00b",
-    urls = ["https://github.com/bazelbuild/platforms/archive/98939346da932eef0b54cf808622f5bb0928f00b.zip"],
-)
-
-# RocksDB
-new_local_repository(
-    name = "rocksdb",
-    build_file = "//:thirdparties/rocksdb.BUILD",
-    path = "thirdparties/rocksdb",
-)
-
-# Hedron's Compile Commands Extractor for Bazel
-# https://github.com/hedronvision/bazel-compile-commands-extractor
-http_archive(
-    name = "hedron_compile_commands",
-
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
-    # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
-        "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
-    ],
-    strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
-)
-load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
-hedron_compile_commands_setup()

bazel/gflags.BUILD

@@ -14,17 +14,21 @@
 #  limitations under the License.
 #
 
-load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
+# Bazel (http://bazel.io/) BUILD file for gflags.
+#
+# See INSTALL.md for instructions for adding gflags to a Bazel workspace.
 
-cc_library(
-    name = "idgenerator",
-    srcs = glob(["*.cpp"]),
-    hdrs = glob(["*.h"]),
-    copts = CURVE_DEFAULT_COPTS,
-    visibility = ["//visibility:public"],
-    deps = [
-        "//external:glog",
-        "//src/common:curve_common",
-        "//src/kvstorageclient:kvstorage_client",
-    ],
+licenses(["notice"])
+
+exports_files(["src/gflags_completions.sh", "COPYING.txt"])
+
+config_setting(
+    name = "x64_windows",
+    values = {"cpu": "x64_windows"},
 )
+
+load(":bazel/gflags.bzl", "gflags_sources", "gflags_library")
+
+(hdrs, srcs) = gflags_sources(namespace=["gflags", "google"])
+gflags_library(hdrs=hdrs, srcs=srcs, threads=0)
+gflags_library(hdrs=hdrs, srcs=srcs, threads=1)

bazel/glog.BUILD

@@ -0,0 +1,98 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+cc_library(
+    name = "glog",
+    srcs = [
+        "config.h",
+        "src/base/commandlineflags.h",
+        "src/base/googleinit.h",
+        "src/base/mutex.h",
+        "src/demangle.cc",
+        "src/demangle.h",
+        "src/logging.cc",
+        "src/raw_logging.cc",
+        "src/signalhandler.cc",
+        "src/symbolize.cc",
+        "src/symbolize.h",
+        "src/utilities.cc",
+        "src/utilities.h",
+        "src/vlog_is_on.cc",
+    ] + glob(["src/stacktrace*.h"]),
+    hdrs = [
+        "src/glog/log_severity.h",
+        "src/glog/logging.h",
+        "src/glog/raw_logging.h",
+        "src/glog/stl_logging.h",
+        "src/glog/vlog_is_on.h",
+    ],
+    copts = [
+        "-Wno-sign-compare",
+        "-U_XOPEN_SOURCE",
+    ],
+    includes = ["./src"],
+    linkopts = ["-lpthread"] + select({
+        ":libunwind": ["-lunwind"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_gflags_gflags//:gflags",
+    ],
+)
+
+config_setting(
+    name = "libunwind",
+    values = {
+        "define": "libunwind=true",
+    },
+)
+
+genrule(
+    name = "run_configure",
+    srcs = [
+        "README",
+        "Makefile.in",
+        "config.guess",
+        "config.sub",
+        "install-sh",
+        "ltmain.sh",
+        "missing",
+        "libglog.pc.in",
+        "src/config.h.in",
+        "src/glog/logging.h.in",
+        "src/glog/raw_logging.h.in",
+        "src/glog/stl_logging.h.in",
+        "src/glog/vlog_is_on.h.in",
+    ],
+    outs = [
+        "config.h",
+        "src/glog/logging.h",
+        "src/glog/raw_logging.h",
+        "src/glog/stl_logging.h",
+        "src/glog/vlog_is_on.h",
+    ],
+    tools = [
+        "configure",
+    ],
+    cmd = "$(location :configure)" +
+          "&& cp -v src/config.h $(location config.h) " +
+          "&& cp -v src/glog/logging.h $(location src/glog/logging.h) " +
+          "&& cp -v src/glog/raw_logging.h $(location src/glog/raw_logging.h) " +
+          "&& cp -v src/glog/stl_logging.h $(location src/glog/stl_logging.h) " +
+          "&& cp -v src/glog/vlog_is_on.h $(location src/glog/vlog_is_on.h) "
+    ,
+)

bazel/gmock.BUILD

@@ -0,0 +1,44 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+cc_library(
+      name = "gtest",
+      srcs = [
+            "googletest/src/gtest-all.cc",
+            "googlemock/src/gmock-all.cc",
+      ],
+      hdrs = glob([
+          "**/*.h",
+          "googletest/src/*.cc",
+          "googlemock/src/*.cc",
+      ]),
+      includes = [
+          "googlemock",
+          "googletest",
+          "googletest/include",
+          "googlemock/include",
+      ],
+      linkopts = ["-pthread"],
+      visibility = ["//visibility:public"],
+  )
+
+cc_library(
+      name = "gtest_main",
+      srcs = ["googlemock/src/gmock_main.cc"],
+      linkopts = ["-pthread"],
+      visibility = ["//visibility:public"],
+      deps = [":gtest"],
+)

bazel/gtest.BUILD

@@ -15,12 +15,16 @@
 #
 
 cc_library(
-    name = "spdlog",
+    name = "main",
+    srcs = glob(
+        ["src/*.cc"],
+        exclude = ["src/gtest-all.cc"]
+    ),
     hdrs = glob([
         "include/**/*.h",
+        "src/*.h"
     ]),
-    defines = ["SPDLOG_FMT_EXTERNAL"],
-    includes = ["include"],
+    copts = ["-Iexternal/gtest/include"],
+    linkopts = ["-pthread"],
     visibility = ["//visibility:public"],
-    deps = ["@fmt//:fmt"],
-)
+)

bazel/leveldb.BUILD

@@ -0,0 +1,98 @@
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+package(default_visibility = ["//visibility:public"])
+
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+SOURCES = ["db/builder.cc",
+         "db/c.cc",
+         "db/dbformat.cc",
+         "db/db_impl.cc",
+         "db/db_iter.cc",
+         "db/dumpfile.cc",
+         "db/filename.cc",
+         "db/log_reader.cc",
+         "db/log_writer.cc",
+         "db/memtable.cc",
+         "db/repair.cc",
+         "db/table_cache.cc",
+         "db/version_edit.cc",
+         "db/version_set.cc",
+         "db/write_batch.cc",
+         "table/block_builder.cc",
+         "table/block.cc",
+         "table/filter_block.cc",
+         "table/format.cc",
+         "table/iterator.cc",
+         "table/merger.cc",
+         "table/table_builder.cc",
+         "table/table.cc",
+         "table/two_level_iterator.cc",
+         "util/arena.cc",
+         "util/bloom.cc",
+         "util/cache.cc",
+         "util/coding.cc",
+         "util/comparator.cc",
+         "util/crc32c.cc",
+         "util/env.cc",
+         "util/env_posix.cc",
+         "util/filter_policy.cc",
+         "util/hash.cc",
+         "util/histogram.cc",
+         "util/logging.cc",
+         "util/options.cc",
+         "util/status.cc",
+         "port/port_posix.cc",
+         "port/port_posix_sse.cc",
+         "helpers/memenv/memenv.cc",
+    ]
+
+cc_library(
+    name = "leveldb",
+    srcs = SOURCES,
+    hdrs = glob([ 
+        "helpers/memenv/*.h",
+        "util/*.h",
+        "port/*.h",
+        "port/win/*.h",
+        "table/*.h",
+        "db/*.h",
+        "include/leveldb/*.h"
+    ],
+    exclude = [
+            "**/*test.*",
+    ]),
+    includes = [
+        "include/",
+    ],
+    copts = [
+        "-fno-builtin-memcmp",
+        "-DLEVELDB_PLATFORM_POSIX=1",
+        "-DLEVELDB_ATOMIC_PRESENT",
+    ],
+    defines = [
+        "LEVELDB_PLATFORM_POSIX",
+    ] + select({
+        ":darwin": ["OS_MACOSX"],
+        "//conditions:default": [],
+    }),
+)

build.sh

@@ -51,16 +51,16 @@ curve_version=${tag_version}+${commit_id}${debug}
 
 
 #step3 执行编译
-# check bazel verion, bazel vesion must = 4.2.2
+# check bazel verion, bazel vesion must = 0.17.2
 bazel_version=`bazel version | grep "Build label" | awk '{print $3}'`
 if [ -z ${bazel_version} ]
 then
-    echo "please install bazel 4.2.2 first"
+    echo "please install bazel 0.17.2 first"
     exit
 fi
-if [ ${bazel_version} != "4.2.2" ]
+if [ ${bazel_version} != "0.17.2" ]
 then
-    echo "bazel version must 4.2.2"
+    echo "bazel version must 0.17.2"
     echo "now version is ${bazel_version}"
     exit
 fi

buildfs.sh

@@ -1,48 +0,0 @@
-#!/bin/sh
-
-cpplint --recursive curvefs
-if [ $? -ne 0 ]
-then
-    echo "cpplint failed"
-    exit
-fi
-
-if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
-then
-    bazelflags=''
-else
-    bazelflags='--copt -faligned-new'
-fi
-
-if [ "$1" = "debug" ]
-then
-DEBUG_FLAG="--compilation_mode=dbg"
-fi
-
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
---define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
--DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
-
-if [ $? -ne 0 ]
-then
-    echo "build curvefs failed"
-    exit
-fi
-echo "build curvefs success"
-
-if [ "$1" = "test" ]
-then
-./bazel-bin/curvefs/test/metaserver/curvefs_metaserver_test
-if [ $? -ne 0 ]
-then
-    echo "metaserver_test failed"
-    exit
-fi
-./bazel-bin/curvefs/test/mds/curvefs_mds_test
-if [ $? -ne 0 ]
-then
-    echo "mds_test failed"
-    exit
-fi
-fi
-echo "end compile"

conf/chunkserver.conf

@@ -54,8 +54,6 @@ chunkserver.snapshot_throttle_throughput_bytes=20971520
 # 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
 # 不是20MB的带宽
 chunkserver.snapshot_throttle_check_cycles=4
-# 限制inflight io数量，一般是5000
-chunkserver.max_inflight_requests=5000
 
 #
 # Testing purpose settings
@@ -93,6 +91,7 @@ copyset.raft_meta_uri=local://./0/copysets  # __CURVEADM_TEMPLATE__ local://${pr
 copyset.raft_snapshot_uri=curve://./0/copysets  # __CURVEADM_TEMPLATE__ curve://${prefix}/data/copysets __CURVEADM_TEMPLATE__
 # copyset回收目录
 copyset.recycler_uri=local://./0/recycler  # __CURVEADM_TEMPLATE__ local://${prefix}/data/recycler __CURVEADM_TEMPLATE__
+copyset.max_inflight_requests=5000
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
 # chunkserver use how many threads to use copyset complete sync. 

conf/chunkserver.conf.example

@@ -50,8 +50,6 @@ chunkserver.snapshot_throttle_throughput_bytes=20971520
 # 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
 # 不是20MB的带宽
 chunkserver.snapshot_throttle_check_cycles=4
-# 限制inflight io数量，一般是5000
-chunkserver.max_inflight_requests=5000
 
 #
 # Testing purpose settings
@@ -89,6 +87,7 @@ copyset.raft_meta_uri=local://./0/copysets
 copyset.raft_snapshot_uri=curve://./0/copysets
 # copyset回收目录
 copyset.recycler_uri=local://./0/recycler
+copyset.max_inflight_requests=5000
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
 # chunkserver use how many threads to use copyset complete sync. 
@@ -100,16 +99,6 @@ copyset.check_retrytimes=3
 copyset.finishload_margin=2000
 # 循环判定copyset是否加载完成的内部睡眠时间
 copyset.check_loadmargin_interval_ms=1000
-# scan copyset interval
-copyset.scan_interval_sec=5
-# the size each scan 4MB
-copyset.scan_size_byte=4194304
-# the follower send scanmap to leader rpc timeout
-copyset.scan_rpc_timeout_ms=1000
-# the follower send scanmap to leader rpc retry times
-copyset.scan_rpc_retry_times=3
-# the follower send scanmap to leader rpc retry interval
-copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
 copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
@@ -145,8 +134,6 @@ curve.root_password=root_password
 curve.config_path=conf/cs_client.conf
 # s3配置文件
 s3.config_path=conf/s3.conf
-# Curve File time to live
-curve.curve_file_timeout_s=30
 
 #
 # Local FileSystem settings
@@ -193,12 +180,6 @@ chunkfilepool.chunk_file_pool_dir=./0/
 chunkfilepool.cpmeta_file_size=4096
 # chunkfilepool get chunk最大重试次数
 chunkfilepool.retry_times=5
-# Enable clean chunk
-chunkfilepool.clean.enable=true
-# The bytes per write for cleaning chunk (max: 1MB)
-chunkfilepool.clean.bytes_per_write=4096
-# The throttle iops for cleaning chunk (4KB/IO)
-chunkfilepool.clean.throttle_iops=500
 
 #
 # WAL file pool

conf/client.conf

@@ -129,9 +129,6 @@ global.fileIOSplitMaxSizeKB=64
 #
 ################# log相关配置 ###############
 #
-# enable logging or not
-global.logging.enable=True
-#
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径
@@ -155,19 +152,6 @@ global.metricDummyServerStartPort=9000
 # 是否关闭健康检查: true/关闭 false/不关闭
 global.turnOffHealthCheck=true
 
-#
-### throttle config
-#
-throttle.enable=false
-
-##### discard configurations #####
-# enable/disable discard
-discard.enable=true
-# discard granularity
-discard.granularity=4096
-# discard cleanup task delay times in millisecond
-discard.taskDelayMs=60000
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -181,3 +165,4 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
+

conf/cs_client.conf

@@ -151,11 +151,3 @@ global.metricDummyServerStartPort=9000
 # session map文件，存储打开文件的filename到path的映射
 #
 global.sessionMapPath=./session_map.json
-
-##### discard configurations #####
-# enable/disable discard
-discard.enable=false
-# discard granularity
-discard.granularity=4096
-# discard cleanup task delay times in millisecond
-discard.taskDelayMs=60000

conf/mds.conf

@@ -17,10 +17,6 @@ mds.etcd.dailtimeoutMs=5000
 mds.etcd.operation.timeoutMs=5000
 # client操作失败可以重试的次数
 mds.etcd.retry.times=3
-# wait dlock timeout
-mds.etcd.dlock.timeoutMs=10000
-# dlock lease timeout
-mds.etcd.dlock.ttlSec=10
 
 #
 # segment分配量统计相关配置
@@ -30,8 +26,6 @@ mds.segment.alloc.periodic.persistInterMs=10000
 # 出错情况下的重试间隔,单位ms
 mds.segment.alloc.retryInterMs=1000
 
-mds.segment.discard.scanIntevalMs=5000
-
 
 # leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s)
 # 该值和etcd集群election timeout相关.
@@ -53,8 +47,6 @@ mds.enable.leader.scheduler=true
 mds.enable.recover.scheduler=true
 # replicaScheduler开关
 mds.enable.replica.scheduler=true
-# Scan scheduler switch
-mds.enable.scan.scheduler=true
 # copysetScheduler 轮次间隔，单位是s
 mds.copyset.scheduler.intervalSec=5
 # replicaScheduler 轮次间隔，单位是s
@@ -63,8 +55,6 @@ mds.replica.scheduler.intervalSec=5
 mds.leader.scheduler.intervalSec=30
 # recoverScheduler 轮次间隔，单位是s
 mds.recover.scheduler.intervalSec=5
-# Scan scheduler run interval (seconds)
-mds.scan.scheduler.intervalSec=60
 # 每块磁盘上operator的并发度
 mds.schduler.operator.concurrent=1
 # leader变更超时时间, 超时后mds从内存移除该operator
@@ -75,8 +65,6 @@ mds.scheduler.remove.limitSec=300
 mds.scheduler.add.limitSec=1800
 # change一个副本超时时间, 超时后mds从内存移除该operator
 mds.scheduler.change.limitSec=1800
-# Scan operator timeout (seconds)
-mds.scheduler.scan.limitSec=180
 # copyset数量极差不能超过均值的百分比
 mds.scheduler.copysetNumRangePercent=0.05
 # chunkserver上copyset的scatte-rwidth不能超过最小值的百分比
@@ -86,16 +74,6 @@ mds.chunkserver.failure.tolerance=3
 # chunkserver启动coolingTimeSec_后才可以作为target leader, 单位是s
 # TODO(lixiaocui): 续得一定程度上与快照的时间间隔方面做到相关
 mds.scheduler.chunkserver.cooling.timeSec=1800
-# ScanScheduler: scan start hour in one day ([0-23])
-mds.scheduler.scan.startHour=0
-# ScanScheduler: scan end hour in one day ([0-23])
-mds.scheduler.scan.endHour=6
-# ScanScheduler: scan interval for the same copyset (seconds)
-mds.scheduler.scan.intervalSec=86400
-# ScanScheduler: maximum number of scan copysets at the same time for every logical pool
-mds.scheduler.scan.concurrent.per.pool=10
-# ScanScheduler: maximum number of scan copysets at the same time for every chunkserver
-mds.scheduler.scan.concurrent.per.chunkserver=1
 
 #
 # 心跳相关配置,单位为ms
@@ -156,7 +134,7 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 # 请求chunkserver上创建copyset重试间隔
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
-mds.topology.UpdateMetricIntervalSec=10
+mds.topology.UpdateMetricIntervalSec=60
 #和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
 #那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
 #具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
@@ -228,18 +206,6 @@ mds.common.logDir=./  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE
 # 单元测试情况下
 # mds.common.logDir=./runlog/
 
-#
-#### throttle options ####
-#
-# iops
-mds.throttle.iopsMin=2000
-mds.throttle.iopsMax=26000
-mds.throttle.iopsPerGB=30
-# bps
-mds.throttle.bpsMinInMB=120
-mds.throttle.bpsMaxInMB=260
-mds.throttle.bpsPerGBInMB=0.3
-
 #
 ## poolset rules
 #

conf/py_client.conf

@@ -145,11 +145,3 @@ global.metricDummyServerStartPort=10000
 # session map文件，存储打开文件的filename到path的映射
 #
 global.sessionMapPath=./session_map.json
-
-##### discard configurations #####
-# enable/disable discard
-discard.enable=false
-# discard granularity
-discard.granularity=4096
-# discard cleanup task delay times in millisecond
-discard.taskDelayMs=60000

conf/s3.conf

@@ -1,32 +1,17 @@
 #
 # S3 options
 #
-# reserved for backward compatible
 s3.nos_address=
-s3.endpoint=
-# reserved for backward compatible
 s3.snapshot_bucket_name=
-s3.bucket_name=
-s3.ak=fake
-s3.sk=fake
-s3.region=us-east-1
+s3.ak=
+s3.sk=
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=false
 s3.user_agent_conf=S3 Browser
-s3.maxConnections=32
-s3.connectTimeout=60000
-s3.requestTimeout=10000
+s3.max_connections=32
+s3.connect_timeout=60000
+s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
-s3.logPrefix=/data/log/curve/aws_
-s3.asyncThreadNum=64
-# throttle
-s3.throttle.iopsTotalLimit=5000
-s3.throttle.iopsReadLimit=5000
-s3.throttle.iopsWriteLimit=5000
-s3.throttle.bpsTotalMB=1280
-s3.throttle.bpsReadMB=1280
-s3.throttle.bpsWriteMB=1280
-s3.useVirtualAddressing=false
-
+s3.loglevel=4
+s3.async_thread_num=64

conf/snap_client.conf

@@ -151,11 +151,3 @@ global.metricDummyServerStartPort=9000
 # session map文件，存储打开文件的filename到path的映射
 #
 global.sessionMapPath=./session_map.json
-
-##### discard configurations #####
-# enable/disable discard
-discard.enable=false
-# discard granularity
-discard.granularity=4096
-# discard cleanup task delay times in millisecond
-discard.taskDelayMs=60000

conf/snapshot_clone_server.conf

@@ -80,10 +80,6 @@ etcd.dailtimeoutMs=5000
 etcd.operation.timeoutMs=5000
 # client操作失败可以重试的次数
 etcd.retry.times=3
-# wait dlock timeout
-etcd.dlock.timeoutMs=10000
-# dlock lease timeout
-etcd.dlock.ttlSec=10
 
 #
 # leader选举相关参数

conf/tools.conf

@@ -6,8 +6,6 @@ mdsDummyPort=6700  # __CURVEADM_TEMPLATE__ ${cluster_mds_dummy_port} __CURVEADM_
 rpcTimeout=500
 # rpc重试次数
 rpcRetryTimes=5
-# the rpc concurrency to chunkserver
-rpcConcurrentNum=10
 # etcd地址
 etcdAddr=127.0.0.1:2379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__
 # snapshot clone server 地址

copts.bzl

@@ -14,151 +14,26 @@
 #  limitations under the License.
 #
 
-#
-# Copyright 2017 The Abseil Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-BASE_FLAGS = [
-    "-DOS_LINUX",
-    "-DSNAPPY",
-    "-DHAVE_SSE42",
-    "-fno-omit-frame-pointer",
-    "-momit-leaf-frame-pointer",
-    "-msse4.2",
-    "-pthread",
+GCC_FLAGS = [
+        "-pthread",
+        "-std=c++11",
+#        "-Werror",
+#        "-Wsign-compare",
+#        "-Wno-unused-parameter",
+#        "-Wno-unused-variable",
+#        "-Woverloaded-virtual",
+#        "-Wnon-virtual-dtor",
+#        "-Wno-missing-field-initializers",
 ]
 
-CXX_FLAGS = [
-    "-std=c++11",
+GCC_TEST_FLAGS = [
+        "-pthread",
+        "-std=c++11",
+#        "-Werror",
+#        "-Wsign-compare",
+#        "-Wno-unused-parameter",
+#        "-Wno-unused-variable",
+#        "-Woverloaded-virtual",
+#        "-Wnon-virtual-dtor",
+#        "-Wno-missing-field-initializers",
 ]
-
-CURVE_GCC_FLAGS = [
-    "-Wall",
-    "-Wextra",
-    "-Wconversion-null",
-    "-Wformat-security",
-    "-Woverlength-strings",
-    "-Wpointer-arith",
-    "-Wundef",
-    "-Wunused-local-typedefs",
-    "-Wunused-result",
-    "-Wvarargs",
-    "-Wvla",
-    "-Wwrite-strings",
-    "-Werror",
-    "-DNOMINMAX",
-]
-
-CURVE_GCC_TEST_FLAGS = [
-    "-Wno-conversion-null",
-    "-Wno-deprecated-declarations",
-    "-Wno-missing-declarations",
-    "-Wno-sign-compare",
-    "-Wno-unused-function",
-    "-Wno-unused-parameter",
-    "-Wno-unused-result",
-]
-
-# FIXME: temporary disabled because triggered in many places
-CURVE_GCC_DISABLED_FLGAS = [
-    "-Wno-error=sign-compare",
-    "-Wno-error=reorder",
-    "-Wno-error=unused-parameter",
-    "-Wno-error=unused-variable",
-    "-Wno-error=deprecated-declarations",
-    "-Wno-error=narrowing",
-]
-
-# FIXME: temporary disabled because triggered in many places
-CURVE_GCC_TEST_DISABLED_FLAGS = [
-    "-Wno-error=unused-but-set-variable",
-    "-Wno-error=vla",
-    "-Wno-error=uninitialized",
-    "-Wno-error=maybe-uninitialized",
-    "-Wno-error=format",
-    "-Wno-error=write-strings",
-    "-Wno-error=missing-field-initializers",
-]
-
-# FIXME: Verify these flags
-CURVE_LLVM_FLAGS = [
-    "-Wall",
-    "-Wextra",
-    "-Wcast-qual",
-    "-Wconversion",
-    "-Wfloat-overflow-conversion",
-    "-Wfloat-zero-conversion",
-    "-Wfor-loop-analysis",
-    "-Wformat-security",
-    "-Wgnu-redeclared-enum",
-    "-Winfinite-recursion",
-    "-Wliteral-conversion",
-    "-Wmissing-declarations",
-    "-Woverlength-strings",
-    "-Wpointer-arith",
-    "-Wself-assign",
-    "-Wshadow",
-    "-Wstring-conversion",
-    "-Wtautological-overlap-compare",
-    "-Wundef",
-    "-Wuninitialized",
-    "-Wunreachable-code",
-    "-Wunused-comparison",
-    "-Wunused-local-typedefs",
-    "-Wunused-result",
-    "-Wvla",
-    "-Wwrite-strings",
-    "-Wno-float-conversion",
-    "-Wno-float-conversion",
-    "-Wno-float-overflow-conversion",
-    "-Wno-shorten-64-to-32",
-    "-Wno-sign-conversion",
-    "-DNOMINMAX",
-]
-
-CURVE_LLVM_TEST_FLAGS = [
-    "-Wno-c99-extensions",
-    "-Wno-deprecated-declarations",
-    "-Wno-missing-noreturn",
-    "-Wno-missing-prototypes",
-    "-Wno-missing-variable-declarations",
-    "-Wno-null-conversion",
-    "-Wno-shadow",
-    "-Wno-shift-sign-overflow",
-    "-Wno-sign-compare",
-    "-Wno-unused-function",
-    "-Wno-unused-member-function",
-    "-Wno-unused-parameter",
-    "-Wno-unused-private-field",
-    "-Wno-unused-template",
-    "-Wno-used-but-marked-unused",
-    "-Wno-zero-as-null-pointer-constant",
-    "-Wno-gnu-zero-variadic-macro-arguments",
-    "-Wbraced-scalar-init",
-]
-
-# FIXME: temporary disabled because triggered in many places
-CURVE_LLVM_DISABLED_FLGAS = [
-    "-Wno-c++11-narrowing",
-]
-
-CURVE_DEFAULT_COPTS = select({
-    "//:clang_compiler": CURVE_LLVM_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_LLVM_DISABLED_FLGAS,
-    "//conditions:default": CURVE_GCC_FLAGS + CXX_FLAGS + BASE_FLAGS + CURVE_GCC_DISABLED_FLGAS,
-})
-
-CURVE_TEST_COPTS = CURVE_DEFAULT_COPTS + select({
-    "//:clang_compiler": CURVE_LLVM_TEST_FLAGS,
-    "//conditions:default": CURVE_GCC_TEST_FLAGS + CURVE_GCC_TEST_DISABLED_FLAGS,
-})

coverage/check_coverage.sh

@@ -0,0 +1,118 @@
+#!/bin/bash 
+
+line_cover_base=75
+mds_branch_base=70
+snapshot_branch_base=70
+client_branch_base=78
+other_branch_base=65
+line_cover_all=`cat coverage/index.html | grep -A 5  "Lines"  | grep % | awk -F "%" '{print $1}' | awk -F '>' '{print $2}' | awk -F '.' '{print $1}'`
+
+if(("$line_cover_all" < "$line_cover_base"))
+then
+    echo "line cover not ok!.";
+    echo $line_cover_all;
+    exit -1
+else
+    echo "line cover ok!.";
+    echo $line_cover_all;    
+fi
+
+for i in `find coverage -type d | grep mds`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > mds.all
+if [ -s mds.all ]; then    
+    mds_branch=`cat mds.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    mds_branch=0
+fi
+if(("$mds_branch" < "$mds_branch_base"))
+then
+    echo "mds_branch cover not ok!.";
+    echo $mds_branch;
+    exit -1
+else
+    echo "mds_branch cover ok!.";
+    echo $mds_branch;    
+fi
+
+for i in `find coverage -type d | grep tools`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > tools.all
+if [ -s tools.all ]; then
+    tools_branch=`cat tools.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    tools_branch=0
+fi
+
+if(("$tools_branch" < "$other_branch_base"))
+then
+    echo "tools_branch cover not ok!.";
+    echo $tools_branch;
+    #exit -1
+else
+    echo "tools_branch cover ok!.";
+    echo $tools_branch;    
+fi
+
+for i in `find coverage -type d | grep common`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > common.all
+if [ -s common.all ]; then
+    common_branch=`cat common.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    common_branch=0
+fi	
+if(("$common_branch" < "$other_branch_base"))
+then
+    echo "common_branch cover not ok!.";
+    echo $common_branch;
+    exit -1
+else
+    echo "common_branch cover ok!.";
+    echo $common_branch;    
+fi
+
+
+for i in `find coverage -type d | grep chunkserver`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > chunkserver.all
+if [ -s chunkserver.all ]; then
+    chunkserver_branch=`cat chunkserver.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    chunkserver_branch=0
+fi
+if(("$chunkserver_branch" < "$other_branch_base"))
+then
+    echo "chunkserver_branch cover not ok!.";
+    echo $chunkserver_branch;
+    exit -1
+else
+    echo "chunkserver_branch cover ok!.";
+    echo $chunkserver_branch;    
+fi
+
+
+for i in `find coverage/client -type d | grep client`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > client.all
+if [ -s chunkserver.all ]; then
+    client_branch=`cat client.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    client_branch=0
+fi
+if(("$client_branch" < "$client_branch_base"))
+then
+    echo "client_branch cover not ok!.";
+    echo $client_branch;
+    exit -1
+else
+    echo "client_branch cover ok!.";
+    echo $client_branch;    
+fi
+
+for i in `find coverage -type d | grep fs`;do for j in $i;do find $j -name index.html | xargs cat | grep -A 5 "Branches" | grep % | awk -F '>' '{print $2}' | awk '{print $1}' | awk -F '.' '{print $1}' | grep -v tr;done;done > sfs.all
+if [ -s sfs.all ]; then
+    sfs_branch=`cat sfs.all | awk '{sum+=$1} END {print sum/NR}' | awk -F '.' '{print $1}'`
+else 
+    sfs_branch=0
+fi
+if(("$sfs_branch" < "$other_branch_base"))
+then
+    echo "sfs_branch cover not ok!.";
+    echo $sfs_branch;
+    #exit -1
+else
+    echo "sfs_branch cover ok!.";
+    echo $sfs_branch;    
+fi
+

coverage/filterbr.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import sys
+
+def skip_comments(lines):
+  state = 0
+  for line in lines:
+    n = len(line)
+    l = ''
+    p = 0
+    while p < n:
+      if state == 0:
+        a = line.find('//', p)
+        b = line.find('/*', p)
+        if a > -1 and (a < b or b == -1):
+          l += line[p:a]
+          p = n
+        elif b > -1 and (b < a or a == -1):
+          l += line[p:b]
+          p = b+2
+          state = 1
+        else:
+          l += line[p:]
+          p = n
+      elif state == 1:
+        a = line.rfind('*/', p)
+        if a == -1:
+          p = n
+        else:
+          p = a + 2
+          state = 0
+    yield l
+
+def cond_lines(lines):
+  state = 0
+  pcnt = 0
+  for nr, line in enumerate(lines, 1):
+    if not line:
+      continue
+    n = len(line)
+    p = 0
+    do_yield = False
+    while p < n:
+      if state == 0:
+#        p = line.strip().startswith('if', p)
+        p = line.find('if', p)
+        if p == -1:
+          p = n
+          continue
+        if (p == 0 or not line[p-1].isalpha()) \
+            and (p+2 == len(line) or not line[p+2].isalpha()):
+          do_yield = True
+          state = 1
+        p += 2
+      elif state == 1:
+        do_yield = True
+        p = line.find('(', p)
+        if p == -1:
+          p = n
+        else:
+          p += 1
+          state = 2
+          pcnt = 1
+      elif state == 2:
+        do_yield = True
+        for p in range(p, n):
+          if line[p] == '(':
+            pcnt += 1
+          elif line[p] == ')':
+            pcnt -= 1
+          if not pcnt:
+            state = 0
+            break
+        p += 1
+    if do_yield:
+      yield nr
+
+def cond_lines_from_file(filename):
+  with open(filename) as f:
+    yield from cond_lines(skip_comments(f))
+
+def filter_lcov_trace(lines):
+  nrs = set()
+  for line in lines:
+    if line.startswith('SF:'):
+      nrs = set(cond_lines_from_file(line[3:-1]))
+    elif line.startswith('BRDA:'):
+      xs = line[5:].split(',')
+      nr = int(xs[0]) if xs else 0
+      if nr not in nrs:
+        continue
+    yield line
+
+def filter_lcov_trace_file(s_filename, d_file):
+  with open(s_filename) as f:
+    for l in filter_lcov_trace(f):
+      print(l, end='', file=d_file)
+
+if __name__ == '__main__':
+  #for l in cond_lines_from_file(sys.argv[1]):
+  #  print(l)
+
+  filter_lcov_trace_file(sys.argv[1], sys.stdout)
+
+  #with open(sys.argv[1]) as f:
+  #  for l in skip_comments(f):
+  #    print(l)
+
+

coverage/gen-coverage-nebd.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' , '*/client_proto/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+

coverage/gen-coverage.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+

coverage/gen-coverage2.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# 2017, Georg Sauthoff <mail@gms.tf>, GPLv3
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import filterbr
+
+log = logging.getLogger(__name__)
+
+#ex_path = [ '/usr/include/*', 'unittest/*', 'lib*/*', '*@exe/*', 'example/*' ]
+ex_path = [ '/usr/include/*', '/usr/local/include/*', '/usr/lib/*', '*/bazel_out/*', '*/k8-dbg/*', 'test/*', '*/test/*', '*/external/*' , '*/include/*' , '*/thirdparties/*' , '*/snapshotcloneserver/*' ]
+
+brflag = ['--rc', 'lcov_branch_coverage=1']
+
+lcov = 'lcov'
+base = os.path.abspath('.')
+
+cov_init_raw = 'coverage_init_raw.info'
+cov_post_raw = 'coverage_post_raw.info'
+cov_init     = 'coverage_init.info'
+cov_post     = 'coverage_post.info'
+cov_br       = 'coverage.info'
+cov          = 'coverage.info'
+report_dir   = 'coverage'
+
+def setup_logging():
+  log_format      = '{rel_secs:6.1f} {lvl}  {message}'
+  log_date_format = '%Y-%m-%d %H:%M:%S'
+
+  class Relative_Formatter(logging.Formatter):
+    level_dict = { 10 : 'DBG',  20 : 'INF', 30 : 'WRN', 40 : 'ERR',
+        50 : 'CRI' }
+    def format(self, rec):
+      rec.rel_secs = rec.relativeCreated/1000.0
+      rec.lvl = self.level_dict[rec.levelno]
+      return super(Relative_Formatter, self).format(rec)
+
+  log = logging.getLogger() # root logger
+  log.setLevel(logging.DEBUG)
+  ch = logging.StreamHandler()
+  ch.setLevel(logging.INFO)
+  ch.setFormatter(Relative_Formatter(log_format, log_date_format, style='{'))
+  log.addHandler(ch)
+
+def mk_arg_parser():
+  p = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Do some stuff',
+        epilog='...')
+  p.add_argument('--html', action='store_true', default=True,
+      help='generate HTML report (default: on)')
+  p.add_argument('--no-html', dest='html', action='store_false',
+      help='disable html report generation')
+  p.add_argument('--filter-br', action='store_true', default=True,
+      help='filter branch coverage data (default: on)')
+  p.add_argument('--no-filter-br', dest='filter_br', action='store_false',
+      help='disable branch filtering')
+  return p
+
+def parse_args(*a):
+  arg_parser = mk_arg_parser()
+  args = arg_parser.parse_args(*a)
+  global cov_br
+  if args.filter_br:
+    cov_br = 'coverage_br.info'
+  return args
+
+def run(*args, **kw):
+  log.info('Executing: ' + ' '.join(map(lambda s:"'"+s+"'", args[0])))
+  return subprocess.run(*args, **kw, check=True)
+
+
+def main(args):
+  run([lcov, '--directory',  'src', '--capture', '-o', cov_post_raw] + brflag )
+  run([lcov, '--directory',  'src', '--capture', '--initial', '-o', cov_init_raw])
+
+  for i, o in [ (cov_init_raw, cov_init), (cov_post_raw, cov_post) ]:
+    run([lcov, '--remove', i] + ex_path + [ '-o', o] + brflag)
+
+  run([lcov, '-a', cov_init, '-a', cov_post, '-o', cov_br] + brflag)
+
+  if args.filter_br:
+    log.info('Filtering branch coverage data ({} -> {})'.format(cov_br, cov))
+    with open(cov, 'w') as f:
+      filterbr.filter_lcov_trace_file(cov_br, f)
+
+  if args.html:
+    shutil.rmtree(report_dir, ignore_errors=True)
+    run(['genhtml', cov, '--branch-coverage', '--ignore-errors', 'source', '-o', report_dir])
+
+  return 0
+
+if __name__ == '__main__':
+  setup_logging()
+  args = parse_args()
+  sys.exit(main(args))
+
+
+

coverage/ut_incremental_check.py

@@ -0,0 +1,226 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+######################################################################
+# Purpose:    calculate UT coverage of git commits' new code
+# Useage:    ./ut_incremental_check.py
+# Version:    Initial    Version    by wahaha02
+######################################################################
+
+__version__ = 'V1.0'
+__author__ = 'wahaha02'
+__date__ = '2016-7-25'
+__doc__ = '''
+PURPOSE:
+calculate UT coverage of git commits' new code
+
+USAGE:
+./ut_incremental_check.py <since>..<until> <monitor_c_files> <lcov_dir> <threshold>
+example: 
+./ut_incremental_check.py "227b032..79196ba" '["source/soda/sp/lssp/i2c-v2/ksource"]' "coverage" 0.6
+
+WORK PROCESS:
+get changed file list between <since> and <until> , filter by <monitor_c_files> options;
+get changed lines per changed file;
+based on <lcov_dir>, search .gcov.html per file, and get uncover lines;
+create report file:ut_incremental_check_report.html and check <threshold> (cover lines/new lines).
+
+UT:
+./ut_incremental_check.py ut
+'''
+
+__todo__ = '''
+TODO LIST:
+    1. support svn
+    2. refactory html report by django web template
+    3. add commit info in html report
+    4. prompt user/commit/date info when mouse point to uncovered line
+    5. ...
+'''
+
+import sys, os, re
+import json
+import commands
+from HTMLParser import HTMLParser
+from pprint import *
+ 
+DEBUG = 0 
+
+class GcovHTMLParser(HTMLParser):
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.uncovers = []
+        self.covers = []
+        self.islineNum = False
+        self.lineNum = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "span":
+            for a in attrs:
+                if a == ('class', 'lineNum'):
+                    self.islineNum = True
+                if a == ('class', 'lineNoCov'):
+                    self.uncovers.append(self.lineNum)
+                if a == ('class', 'lineCov'):
+                    self.covers.append(self.lineNum)
+                    
+    def handle_data(self, data):
+        if self.islineNum:
+            try:
+                self.lineNum = int(data)
+            except:
+                self.lineNum = -1
+            
+    def handle_endtag(self, tag):
+        if tag == "span":
+            self.islineNum = False
+    
+class UTCover(object) :
+    def __init__(self, since_until, monitor, lcov_dir, thresh) :
+        self.since, self.until = since_until.split('..')
+        self.monitor = json.loads(monitor)
+        self.lcov_dir = lcov_dir
+        self.thresh = float(thresh)
+        
+    def get_src(self):
+        # self.since, self.until, self.monitor
+        satus, output = commands.getstatusoutput("git diff --name-only %s %s" %(self.since, self.until))
+        src_files = [f for f in output.split('\n') 
+                        for m in self.monitor if m in f 
+                        if os.path.splitext(f)[1][1:] in ['c', 'cpp']]
+        if DEBUG: pprint(src_files)
+        for i in src_files:
+            if i.find("main") > 0:
+                src_files.remove(i)
+        return src_files
+    
+    def get_change(self, src_files):
+        # self.since, self.until
+        changes = {}
+        for f in src_files:
+            satus, output = commands.getstatusoutput("git log --oneline %s..%s %s | awk '{print $1}'" %(self.since, self.until, f))
+            commits = output.split('\n')
+            cmd = "git blame %s | grep -E '(%s)' | awk  -F' *|)' '{print $6}'" %(f, '|'.join(commits))
+            satus, lines = commands.getstatusoutput(cmd)
+            changes[f] = [ int(i) for i in lines.split('\n') if i.isdigit() ]
+
+            
+        if DEBUG: pprint(changes)
+        return changes
+    
+    def get_ghp(self, f):
+        gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+        if not os.path.exists(gcovfile):
+            return None
+            
+        ghp = GcovHTMLParser()
+        ghp.feed(open(gcovfile, 'r').read())
+
+        return ghp
+            
+    def get_lcov_data(self, changes):
+        # self.lcov_dir
+        uncovers = {}
+        lcov_changes = {}
+        
+        for f, lines in changes.items():
+            #print f
+            f = f.split('/', 1)[1]
+            #f = [c.split('/', 1)[1] for c in f]
+            ghp = self.get_ghp(f)
+            if not ghp:
+                uncovers[f] = lines
+                lcov_changes[f] = lines
+                continue
+        
+            if DEBUG: print f, ghp.uncovers, ghp.covers, lines
+            lcov_changes[f] = sorted(list(set(ghp.uncovers + ghp.covers) & set(lines)))
+            uncov_lines = list(set(ghp.uncovers) & set(lines))
+            if len(uncov_lines) != 0:
+                uncovers[f] = sorted(uncov_lines)
+            ghp.close()    
+        
+        return lcov_changes, uncovers
+    
+    def create_uncover_trs(self, uncovers):
+        tr_format = '''
+    <tr>
+      <td class="coverFile"><a href="%(file)s.gcov.html">%(file)s</a></td>
+      <td class="coverFile">%(uncov_lines)s </td>
+    </tr>
+    
+        '''
+        trs = ''
+        for f,v in uncovers.items():
+            gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+            if os.path.exists(gcovfile):
+                s = ''
+                p = re.compile(r'^<span class="lineNum">\s*(?P<num>\d+)\s*</span>')
+                for line in open(gcovfile, 'r').readlines():
+                    ps = p.search(line)
+                    if ps:
+                        s += '<a name="%s">' %ps.group('num') + line + '</a>'
+                    else:
+                        s += line
+                open(gcovfile, 'w').write(s)    
+
+            data = {'file':f, 'uncov_lines':
+                ", ".join(['<a href="%s.gcov.html#%d">%d</a>' %(f, i, i) for i in v])}
+            trs += tr_format %data
+            
+        return trs
+        
+    def create_report(self, changes, uncovers):
+        change_linenum, uncov_linenum = 0, 0
+        for k,v in changes.items():
+            change_linenum += len(v)
+        for k,v in uncovers.items():
+            uncov_linenum += len(v)
+        
+        cov_linenum = change_linenum - uncov_linenum
+        coverage = round(cov_linenum * 1.0 / change_linenum 
+                            if change_linenum > 0 else 1, 4)
+        
+        template = open('ut_incremental_coverage_report.template', 'r').read()
+        data = {    'cov_lines':cov_linenum,
+                    'change_linenum':change_linenum,
+                    'coverage': coverage * 100,
+                    'uncover_trs': self.create_uncover_trs(uncovers)}
+        open(os.path.join(self.lcov_dir, 'ut_incremental_coverage_report.html'),
+            'w').write(template %data)    
+        
+        return coverage
+    
+    def check(self):
+        # main function
+        src_files = self.get_src()
+        changes = self.get_change(src_files)
+        lcov_changes, uncovers = self.get_lcov_data(changes)
+        return 0 if self.create_report(lcov_changes, uncovers) > self.thresh else -1 
+        
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(0)
+if sys.argv[1] == 'ut':
+    monitor, lcov_dir, threshold = ['["src/"]', "/root/.cache/bazel/_bazel_root/6f45ce5b16fddc736efcfa6b0a4c96a2/execroot/curve/bazel-out/k8-dbg/bin/coverage", 0.95]
+    test1 = ["960fc84f..9a928260", monitor, lcov_dir, threshold]
+    if DEBUG: print "test1: ", test1
+    ut = UTCover(*test1)
+    src_files = ut.get_src()
+    print src_files
+    print "====================="
+    changes = ut.get_change(src_files)
+    print "====================="
+    lcov_changes, uncovers = ut.get_lcov_data(changes)
+    print "====================="
+    print uncovers
+    print "====================="
+    rate = ut.create_report(changes, uncovers)
+    print rate
+    print "===================="
+    
+    
+    sys.exit(0)
+    
+ret = UTCover(*sys.argv[1:]).check()
+sys.exit(ret)
+

coverage/ut_incremental_check_nebd.py

@@ -0,0 +1,252 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+######################################################################
+# Purpose:    calculate UT coverage of git commits' new code
+# Useage:    ./ut_incremental_check.py
+# Version:    Initial    Version    by wahaha02
+######################################################################
+
+__version__ = 'V1.0'
+__author__ = 'wahaha02'
+__date__ = '2016-7-25'
+__doc__ = '''
+PURPOSE:
+calculate UT coverage of git commits' new code
+
+USAGE:
+./ut_incremental_check.py <since>..<until> <monitor_c_files> <lcov_dir> <threshold>
+example: 
+./ut_incremental_check.py "227b032..79196ba" '["source/soda/sp/lssp/i2c-v2/ksource"]' "coverage" 0.6
+
+WORK PROCESS:
+get changed file list between <since> and <until> , filter by <monitor_c_files> options;
+get changed lines per changed file;
+based on <lcov_dir>, search .gcov.html per file, and get uncover lines;
+create report file:ut_incremental_check_report.html and check <threshold> (cover lines/new lines).
+
+UT:
+./ut_incremental_check.py ut
+'''
+
+__todo__ = '''
+TODO LIST:
+    1. support svn
+    2. refactory html report by django web template
+    3. add commit info in html report
+    4. prompt user/commit/date info when mouse point to uncovered line
+    5. ...
+'''
+
+import sys, os, re
+import json
+import commands
+from HTMLParser import HTMLParser
+from pprint import *
+ 
+DEBUG = 0
+
+class GcovHTMLParser(HTMLParser):
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.uncovers = []
+        self.covers = []
+        self.islineNum = False
+        self.lineNum = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "span":
+            for a in attrs:
+                if a == ('class', 'lineNum'):
+                    self.islineNum = True
+                if a == ('class', 'lineNoCov'):
+                    self.uncovers.append(self.lineNum)
+                if a == ('class', 'lineCov'):
+                    self.covers.append(self.lineNum)
+                    
+    def handle_data(self, data):
+        if self.islineNum:
+            try:
+                self.lineNum = int(data)
+            except:
+                self.lineNum = -1
+            
+    def handle_endtag(self, tag):
+        if tag == "span":
+            self.islineNum = False
+    
+class UTCover(object) :
+    def __init__(self, since_until, monitor, lcov_dir, thresh) :
+        self.since, self.until = since_until.split('..')
+        self.monitor = json.loads(monitor)
+        self.lcov_dir = lcov_dir
+        self.thresh = float(thresh)
+        
+    def get_src(self):
+        # self.since, self.until, self.monitor
+        satus, output = commands.getstatusoutput("git diff --name-only %s %s" %(self.since, self.until))
+        src_files = [f for f in output.split('\n') 
+                        for m in self.monitor if m in f 
+                        if os.path.splitext(f)[1][1:] in ['c', 'cpp', 'cc']]
+        exclude = ['src/test']
+        src_files_clean = []
+        for f in src_files:
+            need_excluded = False
+            for e in exclude:
+                if e in f:
+                    need_excluded = True
+                    break
+            if not need_excluded:
+                src_files_clean.append(f)
+        print src_files_clean
+        if DEBUG: pprint(src_files)
+        return src_files_clean
+    
+    def get_change(self, src_files):
+        # self.since, self.until
+        changes = {}
+        for f in src_files:
+            satus, output = commands.getstatusoutput("git log --oneline %s..%s %s | awk '{print $1}'" %(self.since, self.until, f))
+            commits = output.split('\n')
+            cmd = "git blame %s | grep -E '(%s)' | cut -d\) -f1 | awk '{print $NF}'" %(f, '|'.join(commits))
+            print cmd
+            satus, lines = commands.getstatusoutput(cmd)
+            changes[f.split("src/")[1]] = [ int(i) for i in lines.split('\n') if i.isdigit() ]
+
+            
+        if DEBUG: pprint(changes)
+        print changes
+        return changes
+    
+    def get_ghp(self, f):
+        gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+        if not os.path.exists(gcovfile):
+            print "%s does not exist" % gcovfile
+            return None
+            
+        ghp = GcovHTMLParser()
+        ghp.feed(open(gcovfile, 'r').read())
+
+        return ghp
+            
+    def get_lcov_data(self, changes):
+        # self.lcov_dir
+        uncovers = {}
+        lcov_changes = {}
+        
+        for f, lines in changes.items():
+            ghp = self.get_ghp(f)
+            if not ghp:
+                uncovers[f] = lines
+                lcov_changes[f] = lines
+                continue
+        
+            if DEBUG: print f, ghp.uncovers, ghp.covers, lines
+            lcov_changes[f] = sorted(list(set(ghp.uncovers + ghp.covers) & set(lines)))
+            uncov_lines = list(set(ghp.uncovers) & set(lines))
+            if len(uncov_lines) != 0:
+                uncovers[f] = sorted(uncov_lines)
+            ghp.close()    
+        
+        return lcov_changes, uncovers
+    
+    def create_uncover_trs(self, uncovers):
+        tr_format = '''
+    <tr>
+      <td class="coverFile"><a href="%(file)s.gcov.html">%(file)s</a></td>
+      <td class="coverFile">%(uncov_lines)s </td>
+    </tr>
+    
+        '''
+        trs = ''
+        for f,v in uncovers.items():
+            gcovfile = os.path.join(self.lcov_dir, f + '.gcov.html')
+            if os.path.exists(gcovfile):
+                s = ''
+                p = re.compile(r'^<span class="lineNum">\s*(?P<num>\d+)\s*</span>')
+                for line in open(gcovfile, 'r').readlines():
+                    ps = p.search(line)
+                    if ps:
+                        s += '<a name="%s">' %ps.group('num') + line + '</a>'
+                    else:
+                        s += line
+                open(gcovfile, 'w').write(s)    
+
+            data = {'file':f, 'uncov_lines':
+                ", ".join(['<a href="%s.gcov.html#%d">%d</a>' %(f, i, i) for i in v])}
+            trs += tr_format %data
+            
+        return trs
+        
+    def create_report(self, changes, uncovers):
+        change_linenum, uncov_linenum = 0, 0
+        for k,v in changes.items():
+            change_linenum += len(v)
+        for k,v in uncovers.items():
+            uncov_linenum += len(v)
+        
+        cov_linenum = change_linenum - uncov_linenum
+        coverage = round(cov_linenum * 1.0 / change_linenum 
+                            if change_linenum > 0 else 1, 4)
+        
+        template = open('ut_incremental_coverage_report.template', 'r').read()
+        data = {    'cov_lines':cov_linenum,
+                    'change_linenum':change_linenum,
+                    'coverage': coverage * 100,
+                    'uncover_trs': self.create_uncover_trs(uncovers)}
+        open(os.path.join(self.lcov_dir, 'ut_incremental_coverage_report.html'),
+            'w').write(template %data)    
+        
+        return coverage
+    
+    def check(self):
+        # main function
+        src_files = self.get_src()
+        changes = self.get_change(src_files)
+        lcov_changes, uncovers = self.get_lcov_data(changes)
+        return 0 if self.create_report(lcov_changes, uncovers) > self.thresh else 1
+        
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(0)
+if sys.argv[1] == 'ut':
+    monitor, lcov_dir, threshold = ['["source/soda/sp/lssp/i2c-v2/ksource"]', "coverage", 0.8]
+    test1 = ["b2016fdb..11440652", monitor, lcov_dir, threshold]
+    if DEBUG: print "test1: ", test1
+    ut = UTCover(*test1)
+    src_files = ut.get_src()
+    assert(src_files == [])
+    changes = ut.get_change(src_files)
+    assert(changes == {})
+    lcov_changes, uncovers = ut.get_lcov_data(changes)
+    assert(uncovers == {})
+    rate = ut.create_report(changes, uncovers)
+    assert(rate == 1)
+    assert(ut.check() == 0)
+    
+    test2 = [
+        "227b03259b33360e2309274f3927c38457d84dd3..79196baabed99661bd31a201ead6764f23a2884c", 
+        monitor, lcov_dir, threshold]
+    if DEBUG: print "test2: ", test2
+    ut = UTCover(*test2)
+    src_files = ut.get_src()
+    assert(src_files == ['source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c', 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c', 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c', 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c'])
+    changes = ut.get_change(src_files)
+    assert(changes ==  {'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c': [78], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c': [56, 57, 58, 59, 60, 130, 131, 132], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c': [68, 69, 115, 118, 124, 125, 126, 454, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 471, 721], 'source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c': [494, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652]})
+    lcov_changes, uncovers = ut.get_lcov_data(changes)
+    assert( lcov_changes == {'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c': [78], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c': [56, 57, 58, 59, 60, 130, 131, 132], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c': [125, 459, 461, 462, 471], 'source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c': [496, 498, 502, 503, 504, 625, 629, 630, 631, 633, 634, 636, 638, 639, 643, 644, 649, 650]}) 
+    assert(uncovers == {'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_pcie.c': [78], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_cfcuctrl.c': [56, 57, 58, 59, 60, 130, 131, 132], 'source/soda/sp/lssp/i2c-v2/ksource/chips/bsp_i2c_opt.c': [125, 471], 'source/soda/sp/lssp/i2c-v2/ksource/bsp_i2c_dev.c': [502, 503, 504, 643, 644]})
+    rate = ut.create_report(changes, uncovers)
+    assert(0.8 > rate > 0.6)
+    assert(ut.check() == 1)
+    
+    test3 = ['d98b93e705a227389e7cdc4b43252f4194a6cb7a..e8876ff5fe8ee0e61865315a67bd395f5d7f63f7 ',
+            monitor, lcov_dir, threshold]
+    if DEBUG: print "test3: ", test3
+    ut = UTCover(*test3)
+    assert(ut.check() == 0)    
+    
+    sys.exit(0)
+    
+ret = UTCover(*sys.argv[1:]).check()
+sys.exit(ret)
+

curve-ansible/README.md

@@ -1,5 +1,3 @@
-**curve-ansile项目当前已经不进行维护，请使用最新工具curveadm进行部署：https://github.com/opencurve/curveadm**
-
 curve-ansible是用ansible编写的curve高性能存储系统远程部署升级工具，可以很方便的在一台主控机上做到部署升级集群所有节点。
 
 ##  一、软件依赖

curve-ansible/common_tasks/install_with_yum_apt.yml

@@ -45,10 +45,10 @@
     - name: install latest repository
       shell: sudo yum -y install https://extras.getpagespeed.com/release-latest.rpm
       register: repository
-      when: result.rc != 0 and install_result.rc != 0
+      when: install_result.rc != 0
     - name: retry install
       shell: sudo yum -y install {{ yum_lib_name }}
-      when: result.rc != 0 and install_result.rc != 0 and repository.rc == 0
+      when: install_result.rc != 0 and repository.rc != 0
   when:
     - "'CentOS' in distro"
 

curve-ansible/host_vars/localhost.yml

@@ -19,3 +19,4 @@ disk_list:
   - sda
   - sdb
   - sdc
+  - sdd

curve-ansible/roles/clean/files/recycle_chunks.sh

@@ -33,7 +33,7 @@ function chunkfilepoolinfo() {
     fi
 
     # chunkfilepool
-    maxChunkId=`ls $chunkfilepool | grep -oE '^([0-9]+)' | sort -rh | head -n 1`
+    maxChunkId=`ls -vr $chunkfilepool | head -1`
     leftChunkNum=`ls $chunkfilepool | wc -l`
 
     # chunknum to be recycled
@@ -57,7 +57,7 @@ function walpoolinfo() {
         return
     fi
     # walfilepool
-    maxSegmentId=`ls $walfilepool | grep -oE '^([0-9]+)' | sort -rh | head -n 1`
+    maxSegmentId=`ls -vr $walfilepool | head -1`
     leftSegmentNum=`ls $walfilepool | wc -l`
 
     # segmentnum to be recycled
@@ -77,8 +77,7 @@ function recycle_copysets() {
     rsegmentSum=0
 
     # recycle chunks
-    maxChunkId=`ls $chunkfilepool | grep -oE '^([0-9]+)' | sort -rh | head -n 1`
-
+    maxChunkId=`ls -vr $chunkfilepool | head -1`
     for copyset in `ls $2`
     do
         copysetPath=${dataDir}/chunkserver$1/copysets/${copyset}
@@ -95,7 +94,7 @@ function recycle_copysets() {
     echo chunkserver$1 actual recycle under data chunk=$rchunkSum
 
     # recycle wal segments
-    maxSegmentId=`ls $walfilepool | grep -oE '^([0-9]+)' | sort -rh | head -n 1`
+    maxSegmentId=`ls -vr $walfilepool | head -1`
     for copyset in `ls $2`
     do
         copysetPath=${dataDir}/chunkserver$1/copysets/${copyset}
@@ -113,7 +112,7 @@ function recycle_copysets() {
 
     # recycle chunks in raft_snapshot
     rchunkSum=0
-    maxChunkId=`ls $chunkfilepool | grep -oE '^([0-9]+)' | sort -rh | head -n 1`
+    maxChunkId=`ls -vr $chunkfilepool | head -1`
     for copyset in `ls $2`
     do
         copysetPath=${dataDir}/chunkserver$1/copysets/${copyset}
@@ -195,4 +194,4 @@ main() {
     wait
 }
 
-main $@
+main $@

curve-ansible/roles/format_chunkserver/tasks/include/prepare_chunkserver_without_disk_format.yml

@@ -24,5 +24,5 @@
 - name: create data dir and log dir for each chunkserver
   vars:
     end_seq: "{{ chunkserver_num | int - 1}}"
-  shell: sudo mkdir -p {{ chunkserver_data_dir }}/chunkserver{{ item }} && sudo mkdir -p {{ chunkserver_data_dir }}/log/chunkserver{{ item }}
+  shell: mkdir -p {{ chunkserver_data_dir }}/chunkserver{{ item }} && mkdir -p {{ chunkserver_data_dir }}/log/chunkserver{{ item }}
   with_sequence: start=0 end={{ end_seq }}

curve-ansible/roles/generate_config/defaults/main.yml

@@ -18,7 +18,6 @@
 # 通用配置
 curve_root_username: root
 curve_root_password: root_password
-curve_file_timeout_s: 30
 chunk_size: 16777216
 segment_size: 1073741824
 min_file_length: 10737418240
@@ -29,38 +28,27 @@ file_expired_time_us: 5000000
 mds_etcd_dailtimeout_ms: 5000
 mds_etcd_operation_timeout_ms: 5000
 mds_etcd_retry_times: 3
-mds_etcd_dlock_timeout_ms: 10000
-mds_etcd_dlock_ttl_sec: 10
 mds_segment_alloc_periodic_persist_inter_ms: 10000
 mds_segment_alloc_retry_inter_ms: 1000
-mds_segment_discard_scan_interval_ms: 5000
 mds_leader_session_inter_sec: 5
 mds_leader_election_timeout_ms: 0
 mds_enable_copyset_scheduler: true
 mds_enable_leader_scheduler: true
 mds_enable_recover_scheduler: true
 mds_enable_replica_scheduler: true
-mds_enable_scan_scheduler: true
 mds_copyset_scheduler_interval_sec: 5
 mds_replica_scheduler_interval_sec: 5
 mds_leader_scheduler_interval_sec: 30
 mds_recover_scheduler_interval_sec: 5
-mds_scan_scheduler_interval_sec: 60
 mds_schduler_operator_concurrent: 1
 mds_schduler_transfer_limit_sec: 60
 mds_scheduler_remove_limit_sec: 300
 mds_scheduler_add_limit_sec: 1800
 mds_scheduler_change_limit_sec: 1800
-mds_scheduler_scan_limit_sec: 180
 mds_scheduler_copyset_mum_range_percent: 0.05
 mds_schduler_scatterwidth_range_percent: 0.2
 mds_chunkserver_failure_tolerance: 3
 mds_scheduler_chunkserver_cooling_time_sec: 1800
-mds_scheduler_scan_start_hour: 0
-mds_scheduler_scan_end_hour: 6
-mds_scheduler_scan_interval_sec: 259200
-mds_scheduler_scan_concurrent_per_pool: 10
-mds_scheduler_scan_concurrent_per_chunkserver: 1
 mds_heartbeat_interval_ms: 10000
 mds_heartbeat_misstimeout_ms: 30000
 mds_heartbeat_offlinet_imeout_ms: 1800000
@@ -87,12 +75,6 @@ mds_chunkserverclient_rpc_retry_interval_ms: 500
 mds_chunkserverclient_update_leader_retry_times: 5
 mds_chunkserverclient_update_leader_retry_interval_ms: 5000
 mds_common_log_dir: ./
-throttle_iops_min: 2000
-throttle_iops_max: 26000
-throttle_iops_per_GB: 30
-throttle_bps_min_in_MB: 120
-throttle_bps_max_in_MB: 260
-throttle_bps_per_GB_in_MB: 0.3
 
 # chunkserver配置默认值
 chunkserver_enable_external_server: true
@@ -105,7 +87,6 @@ chunkserver_heartbeat_timeout: 5000
 chunkserver_stor_uri: local://./0/
 chunkserver_meta_uri: local://./0/chunkserver.dat
 chunkserver_disk_type: nvme
-chunkserver_max_inflight_requests: 5000
 chunkserver_snapshot_throttle_throughput_bytes: 20971520
 chunkserver_snapshot_throttle_check_cycles: 4
 chunkserver_test_create_testcopyset: false
@@ -123,15 +104,11 @@ chunkserver_copyset_raft_log_uri: curve://./0/copysets
 chunkserver_copyset_raft_meta_uri: local://./0/copysets
 chunkserver_copyset_raft_snapshot_uri: curve://./0/copysets
 chunkserver_copyset_recycler_uri: local://./0/recycler
+chunkserver_copyset_max_inflight_requests: 5000
 chunkserver_copyset_load_concurrency: 10
 chunkserver_copyset_check_retrytimes: 3
 chunkserver_copyset_finishload_margin: 2000
 chunkserver_copyset_check_loadmargin_interval_ms: 1000
-chunkserver_copyset_scan_interval_sec: 5
-chunkserver_copyset_scan_size_byte: 4194304
-chunkserver_copyset_scan_rpc_timeout_ms: 1000
-chunkserver_copyset_scan_rpc_retry_times: 3
-chunkserver_copyset_scan_rpc_retry_interval_us: 100000
 chunkserver_copyset_enable_odsync_when_open_chunkfile: false
 chunkserver_copyset_synctimer_interval_ms: 30000
 chunkserver_copyset_check_syncing_interval_ms: 500
@@ -151,9 +128,6 @@ chunkserver_rconcurrentapply_queuedepth: 1
 chunkserver_chunkfilepool_chunk_file_pool_dir: ./0/
 chunkserver_chunkfilepool_cpmeta_file_size: 4096
 chunkserver_chunkfilepool_retry_times: 5
-chunkserver_chunkfilepool_clean_enable: true
-chunkserver_chunkfilepool_clean_bytes_per_write: 4096
-chunkserver_chunkfilepool_clean_throttle_iops: 500
 walfilepool_use_chunk_file_pool: true
 chunkserver_walfilepool_file_pool_dir: ./0/
 chunkserver_walfilepool_meta_path: ./walfilepool.meta
@@ -194,8 +168,6 @@ snap_clone_backend_ref_func_scan_interval_ms: 3600000
 snap_etcd_dailtimeout_ms: 5000
 snap_etcd_operation_timeout_ms: 5000
 snap_etcd_retry_times: 3
-snap_etcd_dlock_timeout_ms: 10000
-snap_etcd_dlock_ttl_sec: 10
 snap_leader_campagin_prefix: snapshotcloneserverleaderlock
 snap_leader_session_inter_sec: 5
 snap_leader_election_timeout_ms: 0
@@ -239,10 +211,6 @@ client_turn_off_health_check: true
 client_session_map_path: ./session_map.json
 client_closefd_timeout_sec: 300
 client_closefd_time_interval_sec: 600
-client_throttle_enable: false
-client_discard_enable: true
-client_discard_granularity: 4096
-client_discard_task_delay_ms: 60000
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf
@@ -267,19 +235,11 @@ s3_max_connections: 32
 s3_connect_timeout: 60000
 s3_request_timeout: 10000
 s3_loglevel: 4
-s3_logPrefix: /data/log/curve/aws_
 s3_async_thread_num: 64
-s3_throttle_iopsTotalLimit: 5000
-s3_throttle_iopsReadLimit: 5000
-s3_throttle_iopsWriteLimit: 5000
-s3_throttle_bpsTotalLimit: 1280
-s3_throttle_bpsReadLimit: 1280
-s3_throttle_bpsWriteLimit: 1280
 
 # 运维工具默认值
 tool_rpc_timeout: 500
 tool_rpc_retry_times: 5
-tool_rpc_concurrent_num: 10
 
 # snapshotclone_nginx配置
 nginx_docker_internal_port: 80

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -51,7 +51,6 @@ chunkserver.snapshot_throttle_throughput_bytes={{ chunkserver_snapshot_throttle_
 # 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
 # 不是20MB的带宽
 chunkserver.snapshot_throttle_check_cycles={{ chunkserver_snapshot_throttle_check_cycles }}
-chunkserver.max_inflight_requests={{ chunkserver_max_inflight_requests }}
 
 #
 # Testing purpose settings
@@ -89,6 +88,7 @@ copyset.raft_meta_uri={{ chunkserver_copyset_raft_meta_uri }}
 copyset.raft_snapshot_uri={{ chunkserver_copyset_raft_snapshot_uri }}
 # copyset回收目录
 copyset.recycler_uri={{ chunkserver_copyset_recycler_uri }}
+copyset.max_inflight_requests={{ chunkserver_copyset_max_inflight_requests }}
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency={{ chunkserver_copyset_load_concurrency }}
 # 检查copyset是否加载完成出现异常时的最大重试次数
@@ -98,19 +98,9 @@ copyset.check_retrytimes={{ chunkserver_copyset_check_retrytimes }}
 copyset.finishload_margin={{ chunkserver_copyset_finishload_margin }}
 # 循环判定copyset是否加载完成的内部睡眠时间
 copyset.check_loadmargin_interval_ms={{ chunkserver_copyset_check_loadmargin_interval_ms }}
-# scan copyset interval
-copyset.scan_interval_sec={{ chunkserver_copyset_scan_interval_sec }}
-# the size each scan 4MB
-copyset.scan_size_byte={{ chunkserver_copyset_scan_size_byte }}
-# the follower send scanmap to leader rpc timeout
-copyset.scan_rpc_timeout_ms={{ chunkserver_copyset_scan_rpc_timeout_ms }}
-# the follower send scanmap to leader rpc retry times
-copyset.scan_rpc_retry_times={{ chunkserver_copyset_scan_rpc_retry_times }}
-# the follower send scanmap to leader rpc retry interval
-copyset.scan_rpc_retry_interval_us={{ chunkserver_copyset_scan_rpc_retry_interval_us }}
-copyset.enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
-copyset.synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
-copyset.check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
+copyset.copyset_enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
+copyset.copyset_synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
+copyset.copyset_check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
 
 #
 # Clone settings
@@ -136,8 +126,6 @@ curve.root_password={{ curve_root_password }}
 curve.config_path={{ chunkserver_client_config_path }}
 # s3配置文件
 s3.config_path={{ chunkserver_s3_config_path }}
-# Curve File time to live
-curve.curve_file_timeout_s={{ curve_file_timeout_s }}
 
 #
 # Local FileSystem settings
@@ -185,12 +173,6 @@ chunkfilepool.chunk_file_pool_dir={{ chunkserver_chunkfilepool_chunk_file_pool_d
 chunkfilepool.cpmeta_file_size={{ chunkserver_chunkfilepool_cpmeta_file_size }}
 # chunkfilepool get chunk最大重试次数
 chunkfilepool.retry_times=5
-# Enable clean chunk
-chunkfilepool.clean.enable={{ chunkserver_chunkfilepool_clean_enable }}
-# The bytes per write for cleaning chunk (max: 1MB)
-chunkfilepool.clean.bytes_per_write={{ chunkserver_chunkfilepool_clean_bytes_per_write }}
-# The throttle iops for cleaning chunk (4KB/IO)
-chunkfilepool.clean.throttle_iops={{ chunkserver_chunkfilepool_clean_throttle_iops }}
 
 #
 # WAL file pool

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -162,16 +162,3 @@ global.turnOffHealthCheck={{ client_turn_off_health_check }}
 # session map文件，存储打开文件的filename到path的映射
 #
 global.sessionMapPath={{ client_session_map_path }}
-
-#
-### throttle config
-#
-throttle.enable={{ client_throttle_enable }}
-
-##### discard configurations #####
-# enable/disable discard
-discard.enable={{ client_discard_enable }}
-# discard granularity
-discard.granularity={{ client_discard_granularity }}
-# discard cleanup task delay times in millisecond
-discard.taskDelayMs={{ client_discard_task_delay_ms }}

curve-ansible/roles/generate_config/templates/grafana.ini.j2

@@ -1,3 +1,21 @@
+#
+#     Copyright (c) 2020 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
 ##################### Grafana Configuration Example #####################
 #
 # Everything has defaults so you only need to uncomment things you want to

curve-ansible/roles/generate_config/templates/mds.conf.j2

@@ -25,10 +25,6 @@ mds.etcd.dailtimeoutMs={{ mds_etcd_dailtimeout_ms }}
 mds.etcd.operation.timeoutMs={{ mds_etcd_operation_timeout_ms }}
 # client操作失败可以重试的次数
 mds.etcd.retry.times={{ mds_etcd_retry_times }}
-# wait dlock timeout
-mds.etcd.dlock.timeoutMs={{ mds_etcd_dlock_timeout_ms }}
-# dlock lease timeout
-mds.etcd.dlock.ttlSec={{ mds_etcd_dlock_ttl_sec }}
 
 #
 # segment分配量统计相关配置
@@ -38,8 +34,6 @@ mds.segment.alloc.periodic.persistInterMs={{ mds_segment_alloc_periodic_persist_
 # 出错情况下的重试间隔,单位ms
 mds.segment.alloc.retryInterMs={{ mds_segment_alloc_retry_inter_ms }}
 
-mds.segment.discard.scanIntevalMs={{ mds_segment_discard_scan_interval_ms }}
-
 
 # leader竞选时会创建session, 单位是秒(go端代码的接口这个值的单位就是s)
 # 该值和etcd集群election timeout相关.
@@ -61,8 +55,6 @@ mds.enable.leader.scheduler={{ mds_enable_leader_scheduler }}
 mds.enable.recover.scheduler={{ mds_enable_recover_scheduler }}
 # replicaScheduler开关
 mds.enable.replica.scheduler={{ mds_enable_replica_scheduler }}
-# Scan scheduler switch
-mds.enable.scan.scheduler={{ mds_enable_scan_scheduler }}
 # copysetScheduler 轮次间隔，单位是s
 mds.copyset.scheduler.intervalSec={{ mds_copyset_scheduler_interval_sec }}
 # replicaScheduler 轮次间隔，单位是s
@@ -71,8 +63,6 @@ mds.replica.scheduler.intervalSec={{ mds_replica_scheduler_interval_sec }}
 mds.leader.scheduler.intervalSec={{ mds_leader_scheduler_interval_sec }}
 # recoverScheduler 轮次间隔，单位是s
 mds.recover.scheduler.intervalSec={{ mds_recover_scheduler_interval_sec }}
-# Scan scheduler run interval (seconds)
-mds.scan.scheduler.intervalSec={{ mds_scan_scheduler_interval_sec }}
 # 每块磁盘上operator的并发度
 mds.schduler.operator.concurrent={{ mds_schduler_operator_concurrent }}
 # leader变更超时时间, 超时后mds从内存移除该operator
@@ -83,8 +73,6 @@ mds.scheduler.remove.limitSec={{ mds_scheduler_remove_limit_sec }}
 mds.scheduler.add.limitSec={{ mds_scheduler_add_limit_sec }}
 # change一个副本超时时间, 超时后mds从内存移除该operator
 mds.scheduler.change.limitSec={{ mds_scheduler_change_limit_sec }}
-# Scan operator timeout (seconds)
-mds.scheduler.scan.limitSec={{ mds_scheduler_scan_limit_sec }}
 # copyset数量极差不能超过均值的百分比
 mds.scheduler.copysetNumRangePercent={{ mds_scheduler_copyset_mum_range_percent }}
 # chunkserver上copyset的scatte-rwidth不能超过最小值的百分比
@@ -94,16 +82,6 @@ mds.chunkserver.failure.tolerance={{ mds_chunkserver_failure_tolerance }}
 # chunkserver启动coolingTimeSec_后才可以作为target leader, 单位是s
 # TODO(lixiaocui): 续得一定程度上与快照的时间间隔方面做到相关
 mds.scheduler.chunkserver.cooling.timeSec={{ mds_scheduler_chunkserver_cooling_time_sec }}
-# ScanScheduler: scan start hour in one day ([0-23])
-mds.scheduler.scan.startHour={{ mds_scheduler_scan_start_hour }}
-# ScanScheduler: scan end hour in one day ([0-23])
-mds.scheduler.scan.endHour={{ mds_scheduler_scan_end_hour }}
-# ScanScheduler: scan interval for the same copyset (seconds)
-mds.scheduler.scan.intervalSec={{ mds_scheduler_scan_interval_sec }}
-# ScanScheduler: maximum number of scan copysets at the same time for every logical pool
-mds.scheduler.scan.concurrent.per.pool={{ mds_scheduler_scan_concurrent_per_pool }}
-# ScanScheduler: maximum number of scan copysets at the same time for every chunkserver
-mds.scheduler.scan.concurrent.per.chunkserver={{ mds_scheduler_scan_concurrent_per_chunkserver }}
 
 #
 # 心跳相关配置,单位为ms
@@ -230,15 +208,3 @@ mds.snapshotcloneclient.addr={{ snapshot_nginx_vip }}:{{ nginx_docker_external_p
 mds.common.logDir={{ mds_common_log_dir }}
 # 单元测试情况下
 # mds.common.logDir=./runlog/
-
-#
-#### throttle options ####
-#
-# iops
-mds.throttle.iopsMin={{ throttle_iops_min }}
-mds.throttle.iopsMax={{ throttle_iops_max }}
-mds.throttle.iopsPerGB={{ throttle_iops_per_GB }}
-# bps
-mds.throttle.bpsMinInMB={{ throttle_bps_min_in_MB }}
-mds.throttle.bpsMaxInMB={{ throttle_bps_max_in_MB }}
-mds.throttle.bpsPerGBInMB={{ throttle_bps_per_GB_in_MB }}

curve-ansible/roles/generate_config/templates/prometheus.yml.j2

@@ -1,3 +1,21 @@
+#
+#     Copyright (c) 2020 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
 # my global config
 global:
   scrape_interval:     {{ prometheus_scrape_interval }} # Set the scrape interval to every 15 seconds. Default is every 1 minute.

curve-ansible/roles/generate_config/templates/s3.conf.j2

@@ -1,27 +1,17 @@
 #
 # S3 options
 #
-s3.nos_address={{ s3_endpoint }}
-s3.endpoint={{ s3_endpoint }}
-s3.snapshot_bucket_name={{ s3_bucket_name }}
-s3.bucket_name={{ s3_bucket_name }}
+s3.nos_address={{ s3_nos_address }}
+s3.snapshot_bucket_name={{ s3_snapshot_bucket_name }}
 s3.ak={{ s3_ak }}
 s3.sk={{ s3_sk }}
 # http = 0, https = 1
 s3.http_scheme={{ s3_http_scheme }}
 s3.verify_SSL={{ s3_verify_ssl }}
 s3.user_agent_conf={{ s3_user_agent_conf }}
-s3.maxConnections={{ s3_max_connections }}
-s3.connectTimeout={{ s3_connect_timeout }}
-s3.requestTimeout={{ s3_request_timeout }}
+s3.max_connections={{ s3_max_connections }}
+s3.connect_timeout={{ s3_connect_timeout }}
+s3.request_timeout={{ s3_request_timeout }}
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel={{ s3_loglevel }}
-s3.logPrefix={{ s3_logPrefix }}
-s3.asyncThreadNum={{ s3_async_thread_num }}
-# throttle
-s3.throttle.iopsTotalLimit={{ s3_throttle_iopsTotalLimit }}
-s3.throttle.iopsReadLimit={{ s3_throttle_iopsReadLimit }}
-s3.throttle.iopsWriteLimit={{ s3_throttle_iopsWriteLimit }}
-s3.throttle.bpsTotalMB= {{ s3_throttle_bpsTotalLimit }}
-s3.throttle.bpsReadMB= {{ s3_throttle_bpsReadLimit }}
-s3.throttle.bpsWriteMB= {{ s3_throttle_bpsWriteLimit }}
+s3.loglevel={{ s3_loglevel }}
+s3.async_thread_num={{ s3_async_thread_num }}

curve-ansible/roles/generate_config/templates/snapshot_clone_server.conf.j2

@@ -87,10 +87,6 @@ etcd.dailtimeoutMs={{ snap_etcd_dailtimeout_ms }}
 etcd.operation.timeoutMs={{ snap_etcd_operation_timeout_ms }}
 # client操作失败可以重试的次数
 etcd.retry.times={{ snap_etcd_retry_times }}
-# wait dlock timeout
-etcd.dlock.timeoutMs={{ snap_etcd_dlock_timeout_ms }}
-# dlock lease timeout
-etcd.dlock.ttlSec={{ snap_etcd_dlock_ttl_sec }}
 
 #
 # leader选举相关参数

curve-ansible/roles/generate_config/templates/tools.conf.j2

@@ -12,8 +12,6 @@ mdsDummyPort={{ hostvars[groups.mds[0]].mds_dummy_port }}
 rpcTimeout={{ tool_rpc_timeout }}
 # rpc重试次数
 rpcRetryTimes={{ tool_rpc_retry_times }}
-# the rpc concurrency to chunkserver
-rpcConcurrentNum={{ tool_rpc_concurrent_num }}
 # etcd地址
 {% set etcd_address=[] -%}
 {% for host in groups.etcd -%}

curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml

@@ -73,5 +73,5 @@
             file_mode: 0755
           include_tasks: copy_file_to_remote.yml
         - name: enable service
-          shell: sudo systemctl daemon-reload && sudo systemctl enable map_curve_disk.service
+          shell: sudo systemctl daemon-reload && sudo systemctl enable map_curve_disk.service && sudo systemctl start map_curve_disk.service
   when: not install_with_deb | bool

curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2

@@ -164,10 +164,10 @@ ret=`lsblk|grep chunkserver|wc -l`
 for i in `seq 0 $((${ret}-1))`
 do
   curve-format -allocatePercent={{ chunk_alloc_percent }} \
-  -filePoolDir=$dataDir/chunkserver$i/chunkfilepool \
-  -filePoolMetaPath=$dataDir/chunkserver$i/chunkfilepool.meta \
+  -filePoolDir=/data/chunkserver$i/chunkfilepool \
+  -filePoolMetaPath=/data/chunkserver$i/chunkfilepool.meta \
   -fileSize={{ chunk_size }} \
-  -fileSystemPath=$dataDir/chunkserver$i/chunkfilepool  &
+  -fileSystemPath=/data/chunkserver$i/chunkfilepool  &
 done
 wait
 }
@@ -182,10 +182,10 @@ function release_disk_reserved_space {
 
 function deploy_one_walfile_pool {
   curve-format -allocatePercent={{ wal_segment_alloc_percent }} \
-  -filePoolDir=$dataDir/chunkserver$1/walfilepool \
-  -filePoolMetaPath=$dataDir/chunkserver$1/walfilepool.meta \
+  -filePoolDir=/data/chunkserver$1/walfilepool \
+  -filePoolMetaPath=/data/chunkserver$1/walfilepool.meta \
   -fileSize={{ chunkserver_walfilepool_segment_size }} \
-  -fileSystemPath=$dataDir/chunkserver$1/walfilepool  &
+  -fileSystemPath=/data/chunkserver$1/walfilepool  &
 }
 
 

curve-ansible/server.ini

@@ -130,12 +130,11 @@ wait_service_timeout=20
 deploy_dir="${HOME}"
 s3_ak=""
 s3_sk=""
-s3_endpoint=""
-s3_bucket_name=""
+s3_nos_address=""
+s3_snapshot_bucket_name=""
 ansible_ssh_port=22
 curve_root_username=root
 curve_root_password=root_password
-curve_file_timeout_s=30
 lib_install_prefix=/usr/local
 bin_install_prefix=/usr
 ansible_connection=local

curve-snapshotcloneserver-nginx/app/etc/config.lua

@@ -0,0 +1,53 @@
+local _M = {}
+
+-- check all the configure from
+-- https://github.com/upyun/lua-resty-checkups
+
+_M.global = {
+    --  Interval of sending heartbeats to backend servers.
+    checkup_timer_interval = 15,
+    
+    -- If set to true, checkups will sync upstram statuses
+    -- from checkups to Nginx upstream blocks.
+    checkup_shd_sync_enable = true,
+
+    -- Interval of syncing upstream status from checkups
+    -- to Nginx upstream blocks.
+    shd_config_timer_interval = 1,
+}
+
+
+_M.snapshot = {
+    -- Enable or disable heartbeats to servers
+    enable = false,
+
+    -- Cluster type, must be one of general, redis, mysql, http. 
+    typ = "http",
+
+    -- Limits the time during which a request can be responsed,
+    -- likewise nginx proxy_next_upstream_timeout.
+    try_timeout = 30,
+
+    --  Connect timeout to upstream servers.
+    timeout = 15,
+
+    -- Retry count. Default is the number of servers.
+    try = 100,
+    -- If set to true and all the servers in the cluster are failing
+    -- checkups will not mark the last failing server as unavailable(err)
+    -- instead, it will be marked as unstable(still available in next try)
+    protected = true,
+
+    -- cluster configure info
+   cluster = {
+        {   -- level 1
+            servers = {
+                { host = "10.182.26.25", port = 5555 },
+                { host = "10.182.26.17", port = 5555 },
+                { host = "10.182.26.16", port = 5555 },
+            }
+        }
+    }
+}
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/argutils.lua

@@ -0,0 +1,233 @@
+-- Copyright (C) 2015 Jingli Chen (Wine93), UPYUN Inc.
+
+local ngx_find     = ngx.re.find
+local escape_uri   = ngx.escape_uri
+
+local sort   = table.sort
+local insert = table.insert
+local concat = table.concat
+local gsub   = string.gsub
+
+local _M = {
+    _VERSION = "0.04"
+}
+
+
+local function split(s, p)
+    local res = {}
+    gsub(s, "([^" .. p .. "]+)", function(w)
+            if #w > 0 then insert(res, w) end
+    end)
+    return res
+end
+
+
+local function match(s, p, ci)
+    local opts = "jo"
+    if ci == true then opts = "ijo" end
+    return ngx_find(s, p, opts)
+end
+
+
+local function parse_arg(arg)
+    if type(arg) ~= "string" or #arg < 1 then
+        return nil, nil, "invalid argument"
+    end
+
+    local from = arg:find("=")
+    if from then
+        local key = arg:sub(1, from - 1)
+        local val = arg:sub(from + 1)
+        if #key > 0 then
+            return key, val
+        end
+    else  -- only key, like ?a
+        return arg, true
+    end
+end
+
+
+local function insert_arg(t, k, v)
+    if not t[k] then
+        t[k] = v
+        return
+    end
+
+    if type(t[k]) == "table" then
+        insert(t[k], v)
+    else
+        t[k] = { t[k], v }
+    end
+end
+
+
+function _M.LESS(a, b)
+    if a.key == b.key then
+        if tostring(a.val) ~= tostring(b.val) then
+            if type(a.val) == "boolean" then
+                return true
+            elseif type(b.val) ~= "boolean"
+                and tostring(a.val) < tostring(b.val) then
+                return true
+            end
+        end
+    end
+
+    return a.key < b.key
+end
+
+
+function _M.sort_args(args, opts)
+    if type(args) ~= "string" then
+        return nil, "non-string arguments"
+    end
+
+    if type(opts) ~= "table" then opts = {} end
+    local order = opts.order or _M.LESS
+
+    if type(order) ~= "function" then
+        return nil, "non-function order"
+    end
+
+    local sorted_args = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            insert(sorted_args, {
+                key = key, val = val
+            })
+        end
+    end
+
+    -- sort the aguments by order function
+    sort(sorted_args, order)
+
+    local res = {}
+    for _, arg in ipairs(sorted_args) do
+        local key = arg.key
+        local val = arg.val
+        if type(val) == "boolean" then
+            insert(res, key)
+        else
+            insert(res, key .. "=" .. val)
+        end
+    end
+
+    return concat(res, '&')
+end
+
+
+function _M.match_args(args, key_pattern, val_pattern, opts)
+    if type(args) ~= "string" then
+        return nil, "non-string argument"
+    end
+    if type(key_pattern) ~= "string" then
+        return nil, "non-string key regex expression"
+    end
+    if type(val_pattern) ~= "string" then
+        return nil, "non-string val regex expression"
+    end
+
+    -- ci == case insensitive
+    if type(opts) ~= "table" then opts = {} end
+    local key_ci = opts.key_ci
+    local val_ci = opts.val_ci
+    local match_bool_value = opts.match_bool_value
+
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            if type(val) == "boolean" then
+                if match_bool_value
+                and match(key, key_pattern, key_ci) then
+                    insert_arg(res, key, val)
+                end
+            elseif type(val) == "string" then
+                if match(key, key_pattern, key_ci)
+                and match(val, val_pattern, val_ci) then
+                    insert_arg(res, key, val)
+                end
+            end
+        end
+    end
+
+    return res
+end
+
+
+function _M.delete_args(args, key_pattern, val_pattern, opts)
+    if type(args) ~= "string" then
+        return nil, "non-string argument"
+    end
+    if type(key_pattern) ~= "string" then
+        return nil, "non-string key regex expression"
+    end
+    if type(val_pattern) ~= "string" then
+        return nil, "non-string val regex expression"
+    end
+
+    -- ci == case insensitive
+    if type(opts) ~= "table" then opts = {} end
+    local key_ci = opts.key_ci
+    local val_ci = opts.val_ci
+    local delete_bool_value = opts.delete_bool_value
+
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            if type(val) == "boolean" then
+                if not delete_bool_value
+                or not match(key, key_pattern, key_ci) then
+                    insert(res, key)
+                end
+            elseif type(val) == "string" then
+                if not match(key, key_pattern, key_ci)
+                or not match(val, val_pattern, val_ci) then
+                    insert(res, key .. "=" .. val)
+                end
+            end
+        end
+    end
+
+    return concat(res, '&')
+end
+
+
+function _M.escape_args(args)
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        if key then
+            key = escape_uri(key)
+            if type(val) == "boolean" then
+                insert(res, key)
+            elseif type(val) == "string" then
+                val = escape_uri(val)
+                insert(res, key .. "=" .. val)
+            end
+        end
+    end
+
+    return concat(res, '&')
+end
+
+
+function _M.split_args(args)
+    local res = {}
+    local args_a = split(args, '&')
+    for _, arg in ipairs(args_a) do
+        local key, val = parse_arg(arg)
+        insert(res, {key, val})
+    end
+
+    return res
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups.lua

@@ -0,0 +1,5 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local api = require "resty.checkups.api"
+
+return api

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/api.lua

@@ -0,0 +1,319 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local cjson      = require "cjson.safe"
+local lrucache   = require "resty.lrucache.pureffi"
+local heartbeat  = require "resty.checkups.heartbeat"
+local dyconfig   = require "resty.checkups.dyconfig"
+local base       = require "resty.checkups.base"
+local try        = require "resty.checkups.try"
+
+
+local localtime  = ngx.localtime
+local mutex      = ngx.shared.mutex
+local state      = ngx.shared.state
+local shd_config = ngx.shared.config
+local log        = ngx.log
+local now        = ngx.now
+local ERR        = ngx.ERR
+local WARN       = ngx.WARN
+local INFO       = ngx.INFO
+local worker_id  = ngx.worker.id
+local get_phase  = ngx.get_phase
+local str_format = string.format
+local find       = string.find
+local type       = type
+local next       = next
+local pairs      = pairs
+local ipairs     = ipairs
+local pcall      = pcall
+
+
+local _M = {
+    _VERSION = "0.20",
+    STATUS_OK       = base.STATUS_OK,
+    STATUS_UNSTABLE = base.STATUS_UNSTABLE,
+    STATUS_ERR      = base.STATUS_ERR
+}
+
+
+function _M.feedback_status(skey, host, port, failed)
+    local ups = base.upstream.checkups[skey]
+    if not ups then
+        return nil, "unknown skey " .. skey
+    end
+
+    local srv
+    for level, cls in pairs(ups.cluster) do
+        for _, s in ipairs(cls.servers) do
+            if s.host == host and s.port == port then
+                srv = s
+                break
+            end
+        end
+    end
+
+    if not srv then
+        return nil, "unknown host:port" .. host .. ":" .. port
+    end
+
+    base.set_srv_status(skey, srv, failed)
+    return 1
+end
+
+
+function _M.ready_ok(skey, callback, opts, upstream)
+    opts = opts or {}
+    local ups = upstream or base.upstream.checkups[skey]
+    if not ups then
+        return nil, "unknown skey " .. skey
+    end
+
+    return try.try_cluster(skey, callback, opts, ups)
+end
+
+
+function _M.init(config)
+    if not config.global.checkup_shd_sync_enable then
+        return true
+    end
+
+    local skeys = {}
+    for skey, ups in pairs(config) do repeat
+        if type(ups) == "table" and type(ups.cluster) == "table" then
+            for level, cls in pairs(ups.cluster) do
+                base.extract_servers_from_upstream(skey, cls)
+            end
+
+            local key = dyconfig._gen_shd_key(skey)
+
+            local encode_status, dup_ups = pcall(cjson.encode, base.table_dup(ups))
+            if encode_status == false then break end
+
+            local ok, err = shd_config:set(key, dup_ups)
+            if not ok then
+                return nil, err
+            end
+        end
+        skeys[skey] = 1
+    until true end
+
+    local ok, err = shd_config:set(base.SHD_CONFIG_VERSION_KEY, 0)
+    if not ok then
+        return nil, err
+    end
+
+    local ok, err = shd_config:set(base.SKEYS_KEY, cjson.encode(skeys))
+    if not ok then
+        return nil, err
+    end
+
+    return true
+end
+
+
+function _M.prepare_checker(config)
+    base.upstream.start_time = localtime()
+    base.upstream.conf_hash = config.global.conf_hash
+    base.upstream.checkup_timer_interval = config.global.checkup_timer_interval or 5
+    base.upstream.checkup_timer_overtime = config.global.checkup_timer_overtime or 60
+    base.upstream.ups_status_sync_enable = config.global.ups_status_sync_enable
+    base.upstream.ups_status_timer_interval = config.global.ups_status_timer_interval or 5
+    base.upstream.checkup_shd_sync_enable = config.global.checkup_shd_sync_enable
+    base.upstream.shd_config_timer_interval = config.global.shd_config_timer_interval
+        or base.upstream.checkup_timer_interval
+    base.upstream.default_heartbeat_enable = config.global.default_heartbeat_enable
+
+    base.upstream.checkups = {}
+    local cluster_status = lrucache.new(config.global.cdn_lrucache_max_items or 1000)
+    local expired = config.global.cdn_srvs_status_expires or 300
+    base.init_cluster_status(cluster_status, expired)
+
+    for skey, ups in pairs(config) do
+        if type(ups) == "table" and type(ups.cluster) == "table" then
+            base.upstream.checkups[skey] = base.table_dup(ups)
+
+            for level, cls in pairs(base.upstream.checkups[skey].cluster) do
+                base.extract_servers_from_upstream(skey, cls)
+            end
+        end
+    end
+
+    if base.upstream.checkup_shd_sync_enable then
+        base.upstream.shd_config_version = 0
+    end
+
+    base.upstream.initialized = true
+end
+
+
+function _M.get_status()
+    local all_status = {}
+    for skey in pairs(base.upstream.checkups) do
+        all_status["cls:" .. skey] = base.get_upstream_status(skey)
+    end
+    local last_check_time = state:get(base.CHECKUP_LAST_CHECK_TIME_KEY) or cjson.null
+    all_status.last_check_time = last_check_time
+    all_status.checkup_timer_alive = state:get(base.CHECKUP_TIMER_ALIVE_KEY) or false
+    all_status.start_time = base.upstream.start_time
+    all_status.conf_hash = base.upstream.conf_hash or cjson.null
+    all_status.shd_config_version = base.upstream.shd_config_version or cjson.null
+
+    all_status.config_timer = dyconfig.get_timer_key_status()
+
+    return all_status
+end
+
+
+function _M.get_ups_timeout(skey)
+    if not skey then
+        return
+    end
+
+    local ups = base.upstream.checkups[skey]
+    if not ups then
+        return
+    end
+
+    local timeout = ups.timeout or 5
+    return timeout, ups.send_timeout or timeout, ups.read_timeout or timeout
+end
+
+
+function _M.create_checker()
+    local phase = get_phase()
+    if phase ~= "init_worker" then
+        error("create_checker must be called in init_worker phase")
+    end
+
+    if not base.upstream.initialized then
+        log(ERR, "create checker failed, call prepare_checker in init_by_lua")
+        return
+    end
+
+    -- shd config syncer enabled
+    if base.upstream.shd_config_version then
+        dyconfig.create_shd_config_syncer()
+    end
+
+    if base.upstream.ups_status_sync_enable and not base.ups_status_timer_created then
+        local ok, err = ngx.timer.at(0, base.ups_status_checker)
+        if not ok then
+            log(WARN, "failed to create ups_status_checker: ", err)
+            return
+        end
+        base.ups_status_timer_created = true
+    end
+
+    if not worker_id then
+        log(ERR, "ngx_http_lua_module version too low, no heartbeat timer will be created")
+        return
+    elseif worker_id() ~= 0 then
+        return
+    end
+
+    -- only worker 0 will create heartbeat timer
+    local ok, err = ngx.timer.at(0, heartbeat.active_checkup)
+    if not ok then
+        log(WARN, "failed to create timer: ", err)
+        return
+    end
+
+    local ckey = base.CHECKUP_TIMER_KEY
+    local overtime = base.upstream.checkup_timer_overtime
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+end
+
+
+function _M.select_peer(skey, ups, opts)
+    return _M.ready_ok(skey, function(host, port)
+        return { host=host, port=port }
+    end, opts, ups)
+end
+
+
+local function gen_upstream(skey, upstream)
+    local ups = upstream
+    if upstream.cluster then
+        -- all upstream
+        if type(upstream.cluster) ~= "table" then
+            return nil, "cluster invalid"
+        end
+    else
+        -- only servers
+        local dyupstream, err = dyconfig.do_get_upstream(skey)
+        if err then
+            return nil, err
+        end
+
+        dyupstream = dyupstream or {}
+        dyupstream.cluster = upstream
+        ups = dyupstream
+    end
+
+    -- check servers
+    local ok
+    for level, cls in pairs(ups.cluster) do
+        if not cls or not next(cls) then
+            return nil, "can not update empty level"
+        end
+
+        local servers = cls.servers
+        if not servers or not next(servers) then
+            return nil, "can not update empty servers"
+        end
+
+        for _, srv in ipairs(servers) do
+            local ok, err = dyconfig.check_update_server_args(skey, level, srv)
+            if not ok then
+                return nil, err
+            end
+        end
+    end
+
+    return ups
+end
+
+
+function _M.update_upstream(skey, upstream)
+    if not upstream or not next(upstream) then
+        return false, "can not set empty upstream"
+    end
+
+    local lock, err = base.get_lock(base.SKEYS_KEY)
+    if not lock then
+        log(WARN, "failed to acquire the lock: ", err)
+        return false, err
+    end
+
+    local ups, err = gen_upstream(skey, upstream)
+    local ok = false
+    if not err then
+        ok, err = dyconfig.do_update_upstream(skey, ups)
+    end
+
+    base.release_lock(lock)
+
+    return ok, err
+end
+
+
+function _M.delete_upstream(skey)
+    local lock, ok, err
+    lock, err = base.get_lock(base.SKEYS_KEY)
+    if not lock then
+        log(WARN, "failed to acquire the lock: ", err)
+        return false, err
+    end
+
+    ok, err = dyconfig.do_delete_upstream(skey)
+
+    base.release_lock(lock)
+
+    return ok, err
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/base.lua

@@ -0,0 +1,367 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local cjson      = require "cjson.safe"
+local lock       = require "resty.lock"
+
+local str_format = string.format
+local str_sub    = string.sub
+local str_find   = string.find
+local str_match  = string.match
+local tab_insert = table.insert
+local unpack     = unpack
+local tostring   = tostring
+local ipairs     = ipairs
+local pairs      = pairs
+local type       = type
+
+local log        = ngx.log
+local ERR        = ngx.ERR
+local WARN       = ngx.WARN
+local state      = ngx.shared.state
+local now        = ngx.now
+
+
+local _M = {
+    _VERSION = "0.20",
+    STATUS_OK = 0,
+    STATUS_UNSTABLE = 1,
+    STATUS_ERR = 2
+}
+
+local ngx_upstream
+
+local CHECKUP_TIMER_KEY = "checkups:timer"
+_M.CHECKUP_TIMER_KEY = CHECKUP_TIMER_KEY
+local CHECKUP_LAST_CHECK_TIME_KEY = "checkups:last_check_time"
+_M.CHECKUP_LAST_CHECK_TIME_KEY = CHECKUP_LAST_CHECK_TIME_KEY
+local CHECKUP_TIMER_ALIVE_KEY = "checkups:timer_alive"
+_M.CHECKUP_TIMER_ALIVE_KEY = CHECKUP_TIMER_ALIVE_KEY
+local PEER_STATUS_PREFIX = "checkups:peer_status:"
+_M.PEER_STATUS_PREFIX = PEER_STATUS_PREFIX
+local SHD_CONFIG_VERSION_KEY = "config_version"
+_M.SHD_CONFIG_VERSION_KEY = SHD_CONFIG_VERSION_KEY
+local SKEYS_KEY = "checkups:skeys"
+_M.SKEYS_KEY = SKEYS_KEY
+local SHD_CONFIG_PREFIX = "shd_config"
+_M.SHD_CONFIG_PREFIX = SHD_CONFIG_PREFIX
+
+
+local upstream = {}
+_M.upstream = upstream
+local peer_id_dict = {}
+
+local expired
+local cluster_status
+
+local ups_status_timer_created
+_M.ups_status_timer_created = ups_status_timer_created
+
+
+_M.is_tab = function(t) return type(t) == "table" end
+_M.is_str = function(t) return type(t) == "string" end
+_M.is_num = function(t) return type(t) == "number" end
+_M.is_nul = function(t) return t == nil or t == ngx.null end
+
+
+local function _gen_key(skey, srv)
+    return str_format("%s:%s:%d", skey, srv.host, srv.port)
+end
+_M._gen_key = _gen_key
+
+
+local function extract_srv_host_port(name)
+    local from, to = str_find(name, ":")
+    if from then
+        local host = str_sub(name, 1, from - 1)
+        local port = str_sub(name, to + 1)
+        host = str_match(host, "^%d+%.%d+%.%d+%.%d+$")
+        port = str_match(port, "^%d+$")
+        if host and port then
+            return host, port
+        end
+    else
+        local host = str_match(name, "^%d+%.%d+%.%d+%.%d+$")
+        if host then
+            return host, 80
+        end
+    end
+end
+
+
+function _M.init_cluster_status(_cluster_status, _expired)
+    expired = _expired
+    cluster_status = _cluster_status
+end
+
+
+function _M.get_srv_status(skey, srv, id)
+    local server_status = cluster_status:get(skey)
+    if not server_status then
+        return _M.STATUS_OK
+    end
+
+    local srv_key = str_format("%s:%d:%s:%s", srv.host, srv.port, srv.isp or "", id)
+    local srv_status = server_status[srv_key]
+    local fail_timeout = srv.fail_timeout or 10
+
+    if srv_status and srv_status.lastmodify + fail_timeout > now() then
+        return srv_status.status
+    end
+
+    return _M.STATUS_OK
+end
+
+
+function _M.set_srv_status(skey, srv, id, failed, _ups)
+    local server_status = cluster_status:get(skey)
+    if not server_status then
+        server_status = {}
+        cluster_status:set(skey, server_status, expired)
+    end
+
+    -- The default max_fails is 0, which differs from nginx upstream module(1).
+    local max_fails = srv.max_fails or 0
+    local fail_timeout = srv.fail_timeout or 10
+    if max_fails == 0 then  -- disables the accounting of attempts
+        return
+    end
+
+    local time_now = now()
+    local srv_key = str_format("%s:%d:%s:%s", srv.host, srv.port, srv.isp or "", id)
+    local srv_status = server_status[srv_key]
+    if not srv_status then  -- first set
+        srv_status = {
+            status = _M.STATUS_OK,
+            failed_count = 0,
+            lastmodify = time_now
+        }
+        server_status[srv_key] = srv_status
+    elseif srv_status.lastmodify + fail_timeout < time_now then -- srv_status expired
+        srv_status.status = _M.STATUS_OK
+        srv_status.failed_count = 0
+        srv_status.lastmodify = time_now
+    end
+
+    if failed then
+        srv_status.failed_count = srv_status.failed_count + 1
+
+        if srv_status.failed_count >= max_fails then
+            local ups = _ups or upstream.checkups[skey]
+            for level, cls in pairs(ups.cluster) do
+                for _, s in ipairs(cls.servers) do
+                    local k = str_format("%s:%d:%s:%s", s.host, s.port, s.isp or "", id)
+                    local st = server_status[k]
+                    -- not the last ok server
+                    if (not st or st.status == _M.STATUS_OK) and k ~= srv_key then
+                        srv_status.status = _M.STATUS_ERR
+                        srv_status.lastmodify = time_now
+                        return
+                    end
+                end
+            end
+        end
+    end
+end
+
+
+function _M.get_lock(key, timeout)
+    local lock = lock:new("locks", {timeout=timeout})
+    local elapsed, err = lock:lock(key)
+    if not elapsed then
+        log(WARN, "failed to acquire the lock: ", key, ", ", err)
+        return nil, err
+    end
+
+    return lock
+end
+
+
+function _M.release_lock(lock)
+    local ok, err = lock:unlock()
+    if not ok then
+        log(WARN, "failed to unlock: ", err)
+    end
+end
+
+
+function _M.get_peer_status(skey, srv)
+    local peer_key = PEER_STATUS_PREFIX .. _gen_key(skey, srv)
+    local peer_status = state:get(peer_key)
+    return not _M.is_nul(peer_status) and cjson.decode(peer_status) or nil
+end
+
+
+function _M.get_upstream_status(skey)
+    local ups = upstream.checkups[skey]
+    if not ups then
+        return
+    end
+
+    local ups_status = {}
+
+    for level, cls in pairs(ups.cluster) do
+        local servers = cls.servers
+        ups_status[level] = {}
+        if servers and type(servers) == "table" and #servers > 0 then
+            for _, srv in ipairs(servers) do
+                local peer_status = _M.get_peer_status(skey, srv) or {}
+                peer_status.server = _gen_key(skey, srv)
+                peer_status["weight"] = srv.weight
+                peer_status["max_fails"] = srv.max_fails
+                peer_status["fail_timeout"] = srv.fail_timeout
+                if ups.enable == false or (ups.enable == nil and
+                   upstream.default_heartbeat_enable == false) then
+                    peer_status.status = "unchecked"
+                else
+                    if not peer_status.status or
+                       peer_status.status == _M.STATUS_OK then
+                        peer_status.status = "ok"
+                    elseif peer_status.status == _M.STATUS_ERR then
+                        peer_status.status = "err"
+                    else
+                        peer_status.status = "unstable"
+                    end
+                end
+                tab_insert(ups_status[level], peer_status)
+            end
+        end
+    end
+
+    return ups_status
+end
+
+
+function _M.extract_servers_from_upstream(skey, cls)
+    local up_key = cls.upstream
+    if not up_key then
+        return
+    end
+
+    cls.servers = cls.servers or {}
+
+    if not ngx_upstream then
+        local ok
+        ok, ngx_upstream = pcall(require, "ngx.upstream")
+        if not ok then
+            log(ERR, "ngx_upstream_lua module required")
+            return
+        end
+    end
+
+    local ups_backup = cls.upstream_only_backup
+    local srvs_getter = ngx_upstream.get_primary_peers
+    if ups_backup then
+        srvs_getter = ngx_upstream.get_backup_peers
+    end
+    local srvs, err = srvs_getter(up_key)
+    if not srvs and err then
+        log(ERR, "failed to get servers in upstream, key: ", up_key, " err:",  err)
+        return
+    end
+
+    for _, srv in ipairs(srvs) do
+        local host, port = extract_srv_host_port(srv.name)
+        if not host then
+            log(ERR, "invalid server name: ", srv.name)
+            return
+        end
+        peer_id_dict[_gen_key(skey, { host = host, port = port })] = {
+            id = srv.id, backup = ups_backup and true or false}
+        tab_insert(cls.servers, {
+            host = host,
+            port = port,
+            weight = srv.weight,
+            max_fails = srv.max_fails,
+            fail_timeout = srv.fail_timeout,
+        })
+    end
+end
+
+
+function _M.table_dup(ori_tab)
+    if type(ori_tab) ~= "table" then
+        return ori_tab
+    end
+    local new_tab = {}
+    for k, v in pairs(ori_tab) do
+        if type(v) == "table" then
+            new_tab[k] = _M.table_dup(v)
+        else
+            new_tab[k] = v
+        end
+    end
+    return new_tab
+end
+
+
+function _M.ups_status_checker(premature)
+    if premature then
+        return
+    end
+
+    if not ngx_upstream then
+        local ok
+        ok, ngx_upstream = pcall(require, "ngx.upstream")
+        if not ok then
+            log(ERR, "ngx_upstream_lua module required")
+            return
+        end
+    end
+
+    local ups_status = {}
+    local names = ngx_upstream.get_upstreams()
+    -- get current upstream down status
+    for _, name in ipairs(names) do
+        local srvs = ngx_upstream.get_primary_peers(name)
+        for _, srv in ipairs(srvs) do
+            ups_status[srv.name] = srv.down and _M.STATUS_ERR or _M.STATUS_OK
+        end
+
+        srvs = ngx_upstream.get_backup_peers(name)
+        for _, srv in ipairs(srvs) do
+            ups_status[srv.name] = srv.down and _M.STATUS_ERR or _M.STATUS_OK
+        end
+    end
+
+    for skey, ups in pairs(upstream.checkups) do
+        for level, cls in pairs(ups.cluster) do
+            if not cls.upstream then
+                break
+            end
+
+            for _, srv in pairs(cls.servers) do
+                local peer_key = _gen_key(skey, srv)
+                local status_key = PEER_STATUS_PREFIX .. peer_key
+
+                local peer_status, err = state:get(status_key)
+                if peer_status then
+                    local st = cjson.decode(peer_status)
+                    local up_st = ups_status[srv.host .. ':' .. srv.port]
+                    local unstable = st.status == _M.STATUS_UNSTABLE
+                    if (unstable and up_st == _M.STATUS_ERR) or
+                        (not unstable and up_st and st.status ~= up_st) then
+                        local up_id = peer_id_dict[peer_key]
+                        local down = up_st == _M.STATUS_OK
+                        local ok, err = ngx_upstream.set_peer_down(
+                            cls.upstream, up_id.backup, up_id.id, down)
+                        if not ok then
+                            log(ERR, "failed to set peer down", err)
+                        end
+                    end
+                elseif err then
+                    log(WARN, "get peer status error ", status_key, " ", err)
+                end
+            end
+        end
+    end
+
+    local interval = upstream.ups_status_timer_interval
+    local ok, err = ngx.timer.at(interval, _M.ups_status_checker)
+    if not ok then
+        ups_status_timer_created = false
+        log(WARN, "failed to create ups_status_checker: ", err)
+    end
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/consistent_hash.lua

@@ -0,0 +1,99 @@
+-- Copyright (C) 2014-2016, UPYUN Inc.
+
+local floor      = math.floor
+local str_byte   = string.byte
+local tab_sort   = table.sort
+local tab_insert = table.insert
+
+local _M = { _VERSION = "0.11" }
+
+local MOD       = 2 ^ 32
+local REPLICAS  = 20
+local LUCKY_NUM = 13
+
+
+local function hash_string(str)
+    local key = 0
+    for i = 1, #str do
+        key = (key * 31 + str_byte(str, i)) % MOD
+    end
+
+    return key
+end
+
+
+local function init_consistent_hash_state(servers)
+    local weight_sum = 0
+    for _, srv in ipairs(servers) do
+        weight_sum = weight_sum + (srv.weight or 1)
+    end
+
+    local circle, members = {}, 0
+    for index, srv in ipairs(servers) do
+        local key = ("%s:%s"):format(srv.host, srv.port)
+        local base_hash = hash_string(key)
+        for c = 1, REPLICAS * weight_sum do
+            -- TODO: more balance hash
+            local hash = (base_hash * c * LUCKY_NUM) % MOD
+            tab_insert(circle, { hash, index })
+        end
+        members = members + 1
+    end
+
+    tab_sort(circle, function(a, b) return a[1] < b[1] end)
+
+    return { circle = circle, members = members }
+end
+
+
+local function binary_search(circle, key)
+    local size = #circle
+    local st, ed, mid = 1, size
+    while st <= ed do
+        mid = floor((st + ed) / 2)
+        if circle[mid][1] < key then
+            st = mid + 1
+        else
+            ed = mid - 1
+        end
+    end
+
+    return st == size + 1 and 1 or st
+end
+
+
+function _M.next_consistent_hash_server(servers, peer_cb, hash_key)
+    local is_tab = require "resty.checkups.base".is_tab
+    servers.chash = is_tab(servers.chash) and servers.chash
+                    or init_consistent_hash_state(servers)
+
+    local chash = servers.chash
+    if chash.members == 1 then
+        if peer_cb(1, servers[1]) then
+            return servers[1]
+        end
+
+        return nil, "consistent hash: no servers available"
+    end
+
+    local circle = chash.circle
+    local st = binary_search(circle, hash_string(hash_key))
+    local size = #circle
+    local ed = st + size - 1
+    for i = st, ed do  -- TODO: algorithm O(n)
+        local idx = circle[(i - 1) % size + 1][2]
+        if peer_cb(idx, servers[idx]) then
+            return servers[idx]
+        end
+    end
+
+    return nil, "consistent hash: no servers available"
+end
+
+
+function _M.free_consitent_hash_server(srv, failed)
+    return
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/dyconfig.lua

@@ -0,0 +1,290 @@
+local cjson         = require "cjson.safe"
+
+local base          = require "resty.checkups.base"
+
+local worker_id     = ngx.worker.id
+local worker_count  = ngx.worker.count
+local update_time   = ngx.update_time
+local mutex         = ngx.shared.mutex
+local state         = ngx.shared.state
+local shd_config    = ngx.shared.config
+local log           = ngx.log
+local ERR           = ngx.ERR
+local WARN          = ngx.WARN
+local INFO          = ngx.INFO
+
+local str_format    = string.format
+local type          = type
+
+local _M = {
+    _VERSION = "0.11",
+    STATUS_OK = base.STATUS_OK, STATUS_UNSTABLE = base.STATUS_UNSTABLE, STATUS_ERR = base.STATUS_ERR
+}
+
+local function _gen_shd_key(skey)
+    return str_format("%s:%s", base.SHD_CONFIG_PREFIX, skey)
+end
+_M._gen_shd_key = _gen_shd_key
+
+
+local function shd_config_syncer(premature)
+    local ckey = base.CHECKUP_TIMER_KEY .. ":shd_config:" .. worker_id()
+    update_time()
+
+    if premature then
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+
+    local interval = base.upstream.shd_config_timer_interval
+
+    local overtime = base.upstream.checkup_timer_overtime
+
+    local lock, err = base.get_lock(base.SKEYS_KEY)
+    if not lock then
+        log(WARN, "upstream updating, failed to acquire the lock: ", base.SKEYS_KEY, ", ", err)
+        local ok, err = ngx.timer.at(interval, shd_config_syncer)
+        if not ok then
+            log(ERR, "failed to create timer: ", err)
+            local ok, err = mutex:set(ckey, nil)
+            if not ok then
+                log(ERR, "failed to update shm: ", err)
+            end
+        else
+            local ok, err = mutex:set(ckey, 1, overtime)
+            if not ok then
+                log(ERR, "failed to update shm: ", err)
+            end
+        end
+        return
+    end
+
+    local config_version, err = shd_config:get(base.SHD_CONFIG_VERSION_KEY)
+
+    if config_version and config_version ~= base.upstream.shd_config_version then
+        local skeys = shd_config:get(base.SKEYS_KEY)
+        if skeys then
+            skeys = cjson.decode(skeys)
+
+            -- delete skey from upstream
+            for skey, _ in pairs(base.upstream.checkups) do
+                if not skeys[skey] then
+                    base.upstream.checkups[skey] = nil
+                end
+            end
+
+            local success = true
+            for skey, _ in pairs(skeys) do
+                local shd_servers, err = shd_config:get(_gen_shd_key(skey))
+                log(INFO, "get ", skey, " from shm: ", shd_servers, " err: ", err)
+                if shd_servers then
+                    shd_servers = cjson.decode(shd_servers)
+                    base.upstream.checkups[skey] = base.table_dup(shd_servers)
+                elseif err then
+                    success = false
+                    log(WARN, "failed to get from shm: ", err)
+                end
+            end
+
+            if success then
+                base.upstream.shd_config_version = config_version
+            end
+        end
+    elseif err then
+        log(WARN, "failed to get config version from shm")
+    end
+
+    base.release_lock(lock)
+
+
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+
+    local ok, err = ngx.timer.at(interval, shd_config_syncer)
+    if not ok then
+        log(ERR, "failed to create timer: ", err)
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+end
+
+_M.shd_config_syncer = shd_config_syncer
+
+
+function _M.check_update_server_args(skey, level, server)
+    if type(skey) ~= "string" then
+        return false, "skey must be a string"
+    end
+    if type(level) ~= "number" and type(level) ~= "string" then
+        return false, "level must be string or number"
+    end
+    if type(server) ~= "table" then
+        return false, "server must be a table"
+    end
+    if not server.host or not server.port then
+        return false, "no server.host nor server.port found"
+    end
+
+    return true
+end
+
+
+function _M.do_get_upstream(skey)
+    local skeys = shd_config:get(base.SKEYS_KEY)
+    if not skeys then
+        return nil, "no skeys found from shm"
+    end
+
+    local key = _gen_shd_key(skey)
+    local shd_servers, err = shd_config:get(key)
+    if shd_servers then
+        shd_servers = cjson.decode(shd_servers)
+        if type(shd_servers) ~= "table" then
+            return nil
+        end
+
+        return shd_servers
+    elseif err then
+        log(WARN, "failed to get from shm: ", err)
+        return nil, err
+    else
+        log(WARN, "upstream " .. skey .. " not found")
+        return nil
+    end
+end
+
+
+function _M.do_update_upstream(skey, upstream)
+    local skeys = shd_config:get(base.SKEYS_KEY)
+    if not skeys then
+        return false, "no skeys found from shm"
+    end
+
+    skeys = cjson.decode(skeys)
+
+    local new_ver, ok, err
+
+    new_ver, err = shd_config:incr(base.SHD_CONFIG_VERSION_KEY, 1)
+
+    if err then
+        log(WARN, "failed to set new version to shm")
+        return false, err
+    end
+
+    local key = _gen_shd_key(skey)
+    ok, err = shd_config:set(key, cjson.encode(upstream))
+
+    if err then
+        log(WARN, "failed to set new upstream to shm")
+        return false, err
+    end
+
+    -- new skey
+    if not skeys[skey] then
+        skeys[skey] = 1
+        local _, err = shd_config:set(base.SKEYS_KEY, cjson.encode(skeys))
+        if err then
+            log(WARN, "failed to set new skeys to shm")
+            return false, err
+        end
+        log(INFO, "add new skey to upstreams, ", skey)
+    end
+
+    return true
+end
+
+
+function _M.do_delete_upstream(skey)
+    local skeys = shd_config:get(base.SKEYS_KEY)
+    if skeys then
+        skeys = cjson.decode(skeys)
+    else
+        return false, "upstream " .. skey .. " not found"
+    end
+
+    local key = _gen_shd_key(skey)
+    local shd_servers, err = shd_config:get(key)
+    if shd_servers then
+        local new_ver, ok, err
+        new_ver, err = shd_config:incr(base.SHD_CONFIG_VERSION_KEY, 1)
+        if err then
+            log(WARN, "failed to set new version to shm")
+            return false, err
+        end
+
+        ok, err = shd_config:delete(key)
+        if err then
+            log(WARN, "failed to delete servers in shm")
+            return false, err
+        end
+
+        skeys[skey] = nil
+
+        local _, err = shd_config:set(base.SKEYS_KEY, cjson.encode(skeys))
+        if err then
+            log(WARN, "failed to set new skeys to shm")
+            return false, err
+        end
+
+        log(INFO, "delete skey from upstreams, ", skey)
+
+    elseif err then
+        return false, err
+    else
+        return false, "upstream " .. skey .. " not found"
+    end
+
+    return true
+end
+
+
+function _M.create_shd_config_syncer()
+    local ok, err = ngx.timer.at(0, shd_config_syncer)
+    if not ok then
+        log(ERR, "failed to create shd_config timer: ", err)
+        return
+    end
+
+    local overtime = base.upstream.checkup_timer_overtime
+    local ckey = base.CHECKUP_TIMER_KEY .. ":shd_config:" .. worker_id()
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+end
+
+
+function _M.get_timer_key_status()
+    if not worker_count then
+        log(WARN, "can not get worker count, please upgrade lua-nginx-module to 0.9.20 or higher")
+        return
+    end
+
+    local timer_status = {}
+    local count = worker_count()
+    for i=0, count-1 do
+        local key = "worker-" .. i
+        local ckey = base.CHECKUP_TIMER_KEY .. ":shd_config:" .. i
+        local val, err = mutex:get(ckey)
+        if err then
+            timer_status[key] = err
+        elseif val then
+            timer_status[key] = "alive"
+        else
+            timer_status[key] = "dead"
+        end
+    end
+
+    return timer_status
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/heartbeat.lua

@@ -0,0 +1,479 @@
+-- Copyright (C) 2014-2016 UPYUN, Inc.
+
+local cjson         = require "cjson.safe"
+
+local base          = require "resty.checkups.base"
+
+local str_sub       = string.sub
+local lower         = string.lower
+local tab_sort      = table.sort
+local tab_concat    = table.concat
+local tab_insert    = table.insert
+
+local re_gmatch     = ngx.re.gmatch
+local re_find       = ngx.re.find
+local log           = ngx.log
+local localtime     = ngx.localtime
+local ERR           = ngx.ERR
+local WARN          = ngx.WARN
+local now           = ngx.now
+local tcp           = ngx.socket.tcp
+local update_time   = ngx.update_time
+local mutex         = ngx.shared.mutex
+local state         = ngx.shared.state
+
+
+local _M = {
+    _VERSION = "0.11",
+    STATUS_OK = base.STATUS_OK, STATUS_UNSTABLE = base.STATUS_UNSTABLE, STATUS_ERR = base.STATUS_ERR
+}
+
+local resty_redis, resty_mysql
+
+
+local function update_peer_status(srv, sensibility)
+    local peer_key = srv.peer_key
+    local status_key = base.PEER_STATUS_PREFIX .. peer_key
+    local status_str, err = state:get(status_key)
+
+    if err then
+        log(ERR, "get old status ", status_key, " ", err)
+        return
+    end
+
+    local old_status, err
+    if status_str then
+        old_status, err = cjson.decode(status_str)
+        if err then
+            log(WARN, "decode old status error: ", err)
+        end
+    end
+
+    if not old_status then
+        old_status = {
+            status = _M.STATUS_OK,
+            fail_num = 0,
+            lastmodified = localtime(),
+        }
+    end
+
+    local status = srv.status
+    if status == _M.STATUS_OK then
+        if old_status.status ~= _M.STATUS_OK then
+            old_status.lastmodified = localtime()
+            old_status.status = _M.STATUS_OK
+        end
+        old_status.fail_num = 0
+    else  -- status == _M.STATUS_ERR or _M.STATUS_UNSTABLE
+        old_status.fail_num = old_status.fail_num + 1
+
+        if old_status.status ~= status and
+            old_status.fail_num >= sensibility then
+            old_status.status = status
+            old_status.lastmodified = localtime()
+        end
+    end
+
+    for k, v in pairs(srv.statuses) do
+        old_status[k] = v
+    end
+
+    local ok, err = state:set(status_key, cjson.encode(old_status))
+    if not ok then
+        log(ERR, "failed to set new status ", err)
+    end
+end
+
+
+local function update_upstream_status(ups_status, sensibility)
+    if not ups_status then
+        return
+    end
+
+    for _, srv in ipairs(ups_status) do
+        update_peer_status(srv, sensibility)
+    end
+end
+
+
+local heartbeat = {
+    general = function (host, port, ups)
+        local id = host .. ':' .. port
+
+        local sock = tcp()
+        sock:settimeout(ups.timeout * 1000)
+        local ok, err = sock:connect(host, port)
+        if not ok then
+            log(ERR, "failed to connect: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        sock:setkeepalive()
+
+        return _M.STATUS_OK
+    end,
+
+    redis = function (host, port, ups)
+        local id = host .. ':' .. port
+
+        if not resty_redis then
+            local ok
+            ok, resty_redis = pcall(require, "resty.redis")
+            if not ok then
+                log(ERR, "failed to require resty.redis")
+                return _M.STATUS_ERR, "failed to require resty.redis"
+            end
+        end
+
+        local red, err = resty_redis:new()
+        if not red then
+            log(WARN, "failed to new redis: ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        red:set_timeout(ups.timeout * 1000)
+
+        local redis_err = { status = _M.STATUS_ERR, replication = cjson.null }
+        local ok, err = red:connect(host, port)
+        if not ok then
+            log(ERR, "failed to connect redis: ", id, ", ", err)
+            return redis_err, err
+        end
+
+        if ups.password then
+            local ok, err = red:auth(ups.password)
+            if err then
+                log(WARN, "failed to auth to redis:", err)
+            end
+        end
+
+        local res, err = red:ping()
+        if not res then
+            log(ERR, "failed to ping redis: ", id, ", ", err)
+            return redis_err, err
+        end
+
+        local replication = {}
+        local statuses = {
+            status = _M.STATUS_OK ,
+            replication = replication
+        }
+
+        local res, got_all_info = {}, false
+
+        local info, err = red:info("replication")
+        if not info then
+            info, err = red:info()
+            if not info then
+                replication.err = err
+                return statuses
+            end
+
+            got_all_info = true
+        end
+
+        tab_insert(res, info)
+
+        if not got_all_info then
+            local info, err = red:info("server")
+            if info then
+                tab_insert(res, info)
+            end
+        end
+
+        res = tab_concat(res)
+
+        red:set_keepalive(10000, 100)
+
+        local iter, err = re_gmatch(res, [[([a-zA-Z_]+):(.+?)\r\n]], "jo")
+        if not iter then
+            replication.err = err
+            return statuses
+        end
+
+        local replication_field = {
+            role                           = true,
+            master_host                    = true,
+            master_port                    = true,
+            master_link_status             = true,
+            master_link_down_since_seconds = true,
+            master_last_io_seconds_ago     = true,
+        }
+
+        local other_field = {
+            redis_version = true,
+        }
+
+        while true do
+            local m, err = iter()
+            if err then
+                replication.err = err
+                return statuses
+            end
+
+            if not m then
+                break
+            end
+
+            if replication_field[lower(m[1])] then
+                replication[m[1]] = m[2]
+            end
+
+            if other_field[lower(m[1])] then
+                statuses[m[1]] = m[2]
+            end
+        end
+
+        if replication.master_link_status == "down" then
+            statuses.status = _M.STATUS_UNSTABLE
+            statuses.msg = "master link status: down"
+        end
+
+        return statuses
+    end,
+
+    mysql = function (host, port, ups)
+        local id = host .. ':' .. port
+
+        if not resty_mysql then
+            local ok
+            ok, resty_mysql = pcall(require, "resty.mysql")
+            if not ok then
+                log(ERR, "failed to require resty.mysql")
+                return _M.STATUS_ERR, "failed to require resty.mysql"
+            end
+        end
+
+        local db, err = resty_mysql:new()
+        if not db then
+            log(WARN, "failed to new mysql: ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        db:set_timeout(ups.timeout * 1000)
+
+        local ok, err, errno, sqlstate = db:connect{
+            host = host,
+            port = port,
+            database = ups.name,
+            user = ups.user,
+            password = ups.pass,
+            max_packet_size = 1024*1024
+        }
+
+        if not ok then
+            log(ERR, "failed to connect: ", id, ", ", err, ": ", errno, " ", sqlstate)
+            return _M.STATUS_ERR, err
+        end
+
+        db:set_keepalive(10000, 100)
+
+        return _M.STATUS_OK
+    end,
+
+    http = function(host, port, ups)
+        local id = host .. ':' .. port
+
+        local sock, err = tcp()
+        if not sock then
+            log(WARN, "failed to create sock: ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        sock:settimeout(ups.timeout * 1000)
+        local ok, err = sock:connect(host, port)
+        if not ok then
+            log(ERR, "failed to connect: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        local opts = ups.http_opts or {}
+
+        local req = opts.query
+        if not req then
+            sock:setkeepalive()
+            return _M.STATUS_OK
+        end
+
+        local bytes, err = sock:send(req)
+        if not bytes then
+            log(ERR, "failed to send request to: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        local readline = sock:receiveuntil("\r\n")
+        local status_line, err = readline()
+        if not status_line then
+            log(ERR, "failed to receive status line from: ", id, ", ", err)
+            return _M.STATUS_ERR, err
+        end
+
+        local statuses = opts.statuses
+        if statuses then
+            local from, to, err = re_find(status_line,
+                [[^HTTP/\d+\.\d+\s+(\d+)]], "joi", nil, 1)
+            if not from then
+                log(ERR, "bad status line from: ", id, ", ", err)
+                return _M.STATUS_ERR, err
+            end
+
+            local status = str_sub(status_line, from, to)
+            if statuses[status] == false then
+                return _M.STATUS_ERR, "bad status code"
+            end
+        end
+
+        sock:setkeepalive()
+
+        return _M.STATUS_OK
+    end,
+}
+
+
+local function cluster_heartbeat(skey)
+    local ups = base.upstream.checkups[skey]
+    if ups.enable == false or (ups.enable == nil and
+        base.upstream.default_heartbeat_enable == false) then
+        return
+    end
+
+    local ups_typ = ups.typ or "general"
+    local ups_heartbeat = ups.heartbeat
+    local ups_sensi = ups.sensibility or 1
+    local ups_protected = true
+    if ups.protected == false then
+        ups_protected = false
+    end
+
+    ups.timeout = ups.timeout or 5
+
+    local server_count = 0
+    for level, cls in pairs(ups.cluster) do
+        if cls.servers and #cls.servers > 0 then
+            server_count = server_count + #cls.servers
+        end
+    end
+
+    local error_count = 0
+    local unstable_count = 0
+    local srv_available = false
+    local ups_status = {}
+    for level, cls in pairs(ups.cluster) do
+        for _, srv in ipairs(cls.servers) do
+            local peer_key = base._gen_key(skey, srv)
+            local cb_heartbeat = ups_heartbeat or heartbeat[ups_typ] or
+                heartbeat["general"]
+            local statuses, err = cb_heartbeat(srv.host, srv.port, ups)
+
+            local status
+            if type(statuses) == "table" then
+                status = statuses.status
+                statuses.status = nil
+            else
+                status = statuses
+                statuses = {}
+            end
+
+            if not statuses.msg then
+                statuses.msg = err or cjson.null
+            end
+
+            local srv_status = {
+                peer_key = peer_key ,
+                status   = status   ,
+                statuses = statuses ,
+            }
+
+            if status == _M.STATUS_OK then
+                update_peer_status(srv_status, ups_sensi)
+                srv_status.updated = true
+                srv_available = true
+                if next(ups_status) then
+                    for _, v in ipairs(ups_status) do
+                        if v.status == _M.STATUS_UNSTABLE then
+                            v.status = _M.STATUS_ERR
+                        end
+                        update_peer_status(v, ups_sensi)
+                    end
+                    ups_status = {}
+                end
+            end
+
+            if status == _M.STATUS_ERR then
+                error_count = error_count + 1
+                if srv_available then
+                    update_peer_status(srv_status, ups_sensi)
+                    srv_status.updated = true
+                end
+            end
+
+            if status == _M.STATUS_UNSTABLE then
+                unstable_count = unstable_count + 1
+                if srv_available then
+                    srv_status.status = _M.STATUS_ERR
+                    update_peer_status(srv_status, ups_sensi)
+                    srv_status.updated = true
+                end
+            end
+
+            if srv_status.updated ~= true then
+                tab_insert(ups_status, srv_status)
+            end
+        end
+    end
+
+    if next(ups_status) then
+        if error_count == server_count then
+            if ups_protected then
+                ups_status[1].status = _M.STATUS_UNSTABLE
+            end
+        elseif error_count + unstable_count == server_count then
+            tab_sort(ups_status, function(a, b) return a.status < b.status end)
+        end
+
+        update_upstream_status(ups_status, ups_sensi)
+    end
+end
+
+
+function _M.active_checkup(premature)
+    local ckey = base.CHECKUP_TIMER_KEY
+
+    update_time() -- flush cache time
+
+    if premature then
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+
+    for skey in pairs(base.upstream.checkups) do
+        cluster_heartbeat(skey)
+    end
+
+    local interval = base.upstream.checkup_timer_interval
+    local overtime = base.upstream.checkup_timer_overtime
+
+    state:set(base.CHECKUP_LAST_CHECK_TIME_KEY, localtime())
+    state:set(base.CHECKUP_TIMER_ALIVE_KEY, true, overtime)
+
+    local ok, err = mutex:set(ckey, 1, overtime)
+    if not ok then
+        log(WARN, "failed to update shm: ", err)
+    end
+
+    local ok, err = ngx.timer.at(interval, _M.active_checkup)
+    if not ok then
+        log(WARN, "failed to create timer: ", err)
+        local ok, err = mutex:set(ckey, nil)
+        if not ok then
+            log(WARN, "failed to update shm: ", err)
+        end
+        return
+    end
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/round_robin.lua

@@ -0,0 +1,72 @@
+-- Copyright (C) 2014-2016, UPYUN Inc.
+
+local ceil = math.ceil
+
+local _M = { _VERSION = "0.11" }
+
+
+--[[
+parameters:
+    - (table) servers
+    - (function) peer_cb(index, server)
+return:
+    - (table) server
+    - (string) error
+--]]
+function _M.next_round_robin_server(servers, peer_cb)
+    local srvs_cnt = #servers
+
+    if srvs_cnt == 1 then
+        if peer_cb(1, servers[1]) then
+            return servers[1], nil
+        end
+
+        return nil, "round robin: no servers available"
+    end
+
+    -- select round robin server
+    local best
+    local max_weight
+    local weight_sum = 0
+    for idx = 1, srvs_cnt do
+        local srv = servers[idx]
+        -- init round robin state
+        srv.weight = srv.weight or 1
+        srv.effective_weight = srv.effective_weight or srv.weight
+        srv.current_weight = srv.current_weight or 0
+
+        if peer_cb(idx, srv) then
+            srv.current_weight = srv.current_weight + srv.effective_weight
+            weight_sum = weight_sum + srv.effective_weight
+
+            if srv.effective_weight < srv.weight then
+                srv.effective_weight = srv.effective_weight + 1
+            end
+
+            if not max_weight or srv.current_weight > max_weight then
+                max_weight = srv.current_weight
+                best = srv
+            end
+        end
+    end
+
+    if not best then
+        return nil, "round robin: no servers available"
+    end
+
+    best.current_weight = best.current_weight - weight_sum
+
+    return best, nil
+end
+
+
+function _M.free_round_robin_server(srv, failed)
+    if not failed then
+        return
+    end
+
+    srv.effective_weight = ceil((srv.effective_weight or 1) / 2)
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/checkups/try.lua

@@ -0,0 +1,248 @@
+-- Copyright (C) 2014-2016, UPYUN Inc.
+
+local cjson           = require "cjson.safe"
+local round_robin     = require "resty.checkups.round_robin"
+local consistent_hash = require "resty.checkups.consistent_hash"
+local base            = require "resty.checkups.base"
+
+local max        = math.max
+local sqrt       = math.sqrt
+local floor      = math.floor
+local tab_insert = table.insert
+local tostring   = tostring
+
+local update_time     = ngx.update_time
+local now             = ngx.now
+
+local _M = { _VERSION = "0.11" }
+
+local is_tab = base.is_tab
+
+local NEED_RETRY       = 0
+local REQUEST_SUCCESS  = 1
+local EXCESS_TRY_LIMIT = 2
+
+
+local function prepare_callbacks(skey, opts, upstream)
+    local ups = upstream or base.upstream.checkups[skey]
+
+    -- calculate count of cluster and server
+    local cls_keys = {}  -- string key or number level
+    local srvs_cnt = 0
+    if is_tab(opts.cluster_key) then  -- specify try cluster
+        for _, cls_key in ipairs(opts.cluster_key) do
+            local cls = ups.cluster[cls_key]
+            if is_tab(cls) then
+                tab_insert(cls_keys, cls_key)
+                srvs_cnt = srvs_cnt + #cls.servers
+            end
+        end
+    else  -- default try cluster
+        for cls_key, cls in pairs(ups.cluster) do
+            tab_insert(cls_keys, cls_key)
+            srvs_cnt = srvs_cnt + #cls.servers
+        end
+    end
+
+
+    -- get next level cluster
+    local cls_key
+    local cls_index = 0
+    local cls_cnt = #cls_keys
+    local next_cluster_cb = function()
+        cls_index = cls_index + 1
+        if cls_index > cls_cnt then
+            return
+        end
+
+        cls_key = cls_keys[cls_index]
+        return ups.cluster[cls_key]
+    end
+
+
+    -- get next select server
+    local mode = ups.mode
+    local next_server_func = round_robin.next_round_robin_server
+    local key
+    if mode ~= nil then
+        if mode == "hash" then
+            key = opts.hash_key or ngx.var.uri
+        elseif mode == "url_hash" then
+            key = ngx.var.uri
+        elseif mode == "ip_hash" then
+            key = ngx.var.remote_addr
+        end
+
+        next_server_func = consistent_hash.next_consistent_hash_server
+    end
+    local next_server_cb = function(servers, peer_cb)
+        return next_server_func(servers, peer_cb, key)
+    end
+
+
+    -- check whether ther server is available
+    local bad_servers = {}
+    local peer_cb = function(index, srv)
+        local key = ("%s:%s:%s:%s"):format(cls_key, srv.host, srv.port, srv.isp or "")
+        -- if bad_servers[key] then
+        --     return false
+        -- end
+
+        if ups.enable == false or (ups.enable == nil
+            and base.upstream.default_heartbeat_enable == false)
+        then
+            return base.get_srv_status(skey, srv, ups.id or "") == base.STATUS_OK
+        end
+
+        local peer_status = base.get_peer_status(skey, srv)
+        if (not peer_status or peer_status.status ~= base.STATUS_ERR)
+            and base.get_srv_status(skey, srv, ups.id or "") == base.STATUS_OK
+        then
+            return true
+        end
+    end
+
+    local retry_sleep = 0.05
+
+    -- check whether need retry
+    local statuses
+    if ups.typ == "http" and is_tab(ups.http_opts) then
+        statuses = ups.http_opts.statuses
+    end
+    local try_cnt = 0
+    local try_limit = opts.try or ups.try or srvs_cnt
+    local retry_cb = function(res)
+        if is_tab(res) and res.status and is_tab(statuses) then
+            if statuses[tostring(res.status)] ~= false then
+                return REQUEST_SUCCESS
+            end
+        elseif res then
+            return REQUEST_SUCCESS
+        end
+
+        try_cnt = try_cnt + 1
+        if try_cnt >= try_limit then
+            return EXCESS_TRY_LIMIT
+        end
+
+        ngx.sleep(try_cnt * retry_sleep)
+
+        return NEED_RETRY
+    end
+
+
+    -- check whether try_time has over amount_request_time
+    local try_time = 0
+    local try_time_limit = opts.try_timeout or ups.try_timeout or 0
+    local try_time_cb = function(this_time_try_time)
+        try_time = try_time + this_time_try_time
+        if try_time_limit == 0 then
+            return NEED_RETRY
+        elseif try_time >= try_time_limit then
+            return EXCESS_TRY_LIMIT
+        end
+
+        return NEED_RETRY
+    end
+
+
+    -- set some status
+    local free_server_func = round_robin.free_round_robin_server
+    if mode == "hash" then
+        free_server_func = consistent_hash.free_consitent_hash_server
+    end
+    local set_status_cb = function(srv, failed)
+        local key = ("%s:%s:%s:%s"):format(cls_key, srv.host, srv.port, srv.isp or "")
+        -- bad_servers[key] = failed
+        base.set_srv_status(skey, srv, ups.id or "", failed, ups)
+        free_server_func(srv, failed)
+    end
+
+
+    return {
+        next_cluster_cb = next_cluster_cb,
+        next_server_cb = next_server_cb,
+        retry_cb = retry_cb,
+        peer_cb = peer_cb,
+        set_status_cb = set_status_cb,
+        try_time_cb = try_time_cb,
+    }
+end
+
+
+local function copy_servers(servers)
+    local _servers = {}
+
+    for _, srv in ipairs(servers) do
+        table.insert(_servers, {host = srv.host, port = srv.port})
+    end
+
+    return _servers
+end
+
+
+--[[
+parameters:
+    - (string) skey
+    - (function) request_cb(host, port)
+    - (table) opts
+        - (number) try
+        - (table) cluster_key
+        - (string) hash_key
+return:
+    - (string) result
+    - (string) error
+--]]
+function _M.try_cluster(skey, request_cb, opts, upstream)
+    local callbacks = prepare_callbacks(skey, opts, upstream)
+
+    local next_cluster_cb = callbacks.next_cluster_cb
+    local next_server_cb  = callbacks.next_server_cb
+    local peer_cb         = callbacks.peer_cb
+    local retry_cb        = callbacks.retry_cb
+    local set_status_cb   = callbacks.set_status_cb
+    local try_time_cb     = callbacks.try_time_cb
+
+    -- iter servers function
+    local itersrvs = function(servers, peer_cb)
+        return function() return next_server_cb(servers, peer_cb) end
+    end
+
+    local res, err = nil, "no servers available"
+    repeat
+        -- get next level/key cluster
+        local cls = next_cluster_cb()
+        if not cls then
+            break
+        end
+
+        local loop_servers = copy_servers(cls.servers)
+        for srv, err in itersrvs(loop_servers, peer_cb) do
+            -- exec request callback by server
+            local start_time = now()
+            local args = opts.args or { srv }
+            res, err = request_cb(srv.host, srv.port, unpack(args))
+
+            local feedback = retry_cb(res)
+
+            -- check whether need retry
+            local end_time = now()
+            local delta_time = end_time - start_time
+
+            set_status_cb(srv, feedback ~= REQUEST_SUCCESS) -- set some status
+            if feedback ~= NEED_RETRY then
+                return res, err
+            end
+
+            local feedback_try_time = try_time_cb(delta_time)
+            if feedback_try_time ~= NEED_RETRY then
+                return nil, "try_timeout excceed"
+            end
+       end
+    until false
+
+    return res, err
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/lib/resty/uuid.lua

@@ -0,0 +1,340 @@
+--- jit-uuid
+-- Fast and dependency-free UUID library for LuaJIT/ngx_lua.
+-- @module jit-uuid
+-- @author Thibault Charbonnier
+-- @license MIT
+-- @release 0.0.5
+
+local bit = require 'bit'
+local tohex = bit.tohex
+local band = bit.band
+local bor = bit.bor
+
+local _M = {
+    _VERSION = '0.0.5'
+}
+
+----------
+-- seeding
+----------
+
+--- Seed the random number generator.
+-- Under the hood, this function calls `math.randomseed`.
+-- It makes sure to use the most appropriate seeding technique for
+-- the current environment, guaranteeing a unique seed.
+--
+-- To guarantee unique UUIDs, you must have correctly seeded
+-- the Lua pseudo-random generator (with `math.randomseed`).
+-- You are free to seed it any way you want, but this function
+-- can do it for you if you'd like, with some added guarantees.
+--
+-- @param[type=number] seed (Optional) A seed to use. If none given, will
+-- generate one trying to use the most appropriate technique.
+-- @treturn number `seed`: the seed given to `math.randomseed`.
+-- @usage
+-- local uuid = require 'resty.jit-uuid'
+-- uuid.seed()
+--
+-- -- in ngx_lua, seed in the init_worker context:
+-- init_worker_by_lua {
+--   local uuid = require 'resty.jit-uuid'
+--   uuid.seed()
+-- }
+function _M.seed(seed)
+    if not seed then
+        if ngx then
+            seed = ngx.time() + ngx.worker.pid()
+        elseif package.loaded['socket'] and package.loaded['socket'].gettime then
+            seed = package.loaded['socket'].gettime()*10000
+        else
+            seed = os.time()
+        end
+    end
+    math.randomseed(seed)
+    return seed
+end
+
+-------------
+-- validation
+-------------
+
+do
+    if ngx and string.find(ngx.config.nginx_configure(),'--with-pcre-jit',nil,true) then
+        local type = type
+        local re_find = ngx.re.find
+        local regex = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'
+
+        --- Validate a string as a UUID.
+        -- To be considered valid, a UUID must be given in its canonical
+        -- form (hexadecimal digits including the hyphen characters).
+        -- This function validates UUIDs disregarding their generation algorithm,
+        -- and in a case-insensitive manner, but checks the variant field.
+        --
+        -- Use JIT PCRE if available in OpenResty or fallbacks on Lua patterns.
+        --
+        -- @param[type=string] str String to verify.
+        -- @treturn boolean `valid`: true if valid UUID, false otherwise.
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- uuid.is_valid 'cbb297c0-a956-486d-ad1d-f9bZZZZZZZZZ' --> false
+        -- uuid.is_valid 'cbb297c0-a956-486d-dd1d-f9b42df9465a' --> false (invalid variant)
+        -- uuid.is_valid 'cbb297c0a956486dad1df9b42df9465a'     --> false (no dashes)
+        -- uuid.is_valid 'cbb297c0-a956-486d-ad1d-f9b42df9465a' --> true
+        function _M.is_valid(str)
+            -- it has proven itself efficient to first check the length with an
+            -- evenly distributed set of valid and invalid uuid lengths.
+            if type(str) ~= 'string' or #str ~= 36 then return false end
+            return re_find(str, regex, 'ioj') ~= nil
+        end
+
+    else
+        local match = string.match
+        local d = '[0-9a-fA-F]'
+        local p = '^'..table.concat({
+            d:rep(8),
+            d:rep(4),
+            d:rep(4),
+            '[89ab]'..d:rep(3),
+            d:rep(12)
+        }, '%-')..'$'
+
+        function _M.is_valid(str)
+            if type(str) ~= 'string' or #str ~= 36 then return false end
+            return match(str, p) ~= nil
+        end
+
+    end
+end
+
+----------------
+-- v4 generation
+----------------
+
+do
+    local fmt = string.format
+    local random = math.random
+
+    --- Generate a v4 UUID.
+    -- v4 UUIDs are created from randomly generated numbers.
+    --
+    -- @treturn string `uuid`: a v4 (randomly generated) UUID.
+    -- @usage
+    -- local uuid = require 'resty.jit-uuid'
+    --
+    -- local u1 = uuid()             ---> __call metamethod
+    -- local u2 = uuid.generate_v4()
+    function _M.generate_v4()
+        return fmt('%s%s%s%s-%s%s-%s%s-%s%s-%s%s%s%s%s%s',
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(bor(band(random(0, 255), 0x0F), 0x40), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(bor(band(random(0, 255), 0x3F), 0x80), 2),
+        tohex(random(0, 255), 2),
+
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2),
+        tohex(random(0, 255), 2))
+    end
+end
+
+----------------
+-- v3/v5 generation
+----------------
+
+do
+    if ngx then
+        local tonumber = tonumber
+        local gmatch = string.gmatch
+        local type = type
+        local char = string.char
+        local fmt = string.format
+        local sub = string.sub
+
+        local function factory(namespace, hash_fn)
+            if not _M.is_valid(namespace) then
+                return nil, 'namespace must be a valid UUID'
+            end
+
+            local binary = ''
+            local iter = gmatch(namespace, '([%a%d][%a%d])') -- pattern faster than PCRE without resty.core
+            while true do
+                local m = iter()
+                if not m then break end
+                binary = binary..char(tonumber(m, 16)) -- no noticeable improvement with buffer table
+            end
+
+            return function(name)
+                if type(name) ~= 'string' then
+                    return nil, 'name must be a string'
+                end
+
+                local hash, ver, var = hash_fn(binary, name)
+
+                return fmt('%s-%s-%s%s-%s%s-%s', sub(hash, 1, 8),
+                sub(hash, 9, 12),
+                ver,
+                sub(hash, 15, 16),
+                var,
+                sub(hash, 19, 20),
+                sub(hash, 21, 32))
+            end
+        end
+
+        --- Instanciate a v3 UUID factory.
+        -- @function factory_v3
+        -- Creates a closure generating namespaced v3 UUIDs.
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @treturn function `factory`: a v3 UUID generator.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local fact = assert(uuid.factory_v3('e6ebd542-06ae-11e6-8e82-bba81706b27d'))
+        --
+        -- local u1 = fact('hello')
+        -- ---> 3db7a435-8c56-359d-a563-1b69e6802c78
+        --
+        -- local u2 = fact('foobar')
+        -- ---> e8d3eeba-7723-3b72-bbc5-8f598afa6773
+        do
+            local md5 = ngx.md5
+
+            local function v3_hash(binary, name)
+                local hash = md5(binary..name)
+                return  hash,
+                tohex(bor(band(tonumber(sub(hash, 13, 14), 16), 0x0F), 0x30), 2),
+                tohex(bor(band(tonumber(sub(hash, 17, 18), 16), 0x3F), 0x80), 2)
+            end
+
+            function _M.factory_v3(namespace)
+                return factory(namespace, v3_hash)
+            end
+        end
+
+        --- Instanciate a v5 UUID factory.
+        -- @function factory_v5
+        -- Creates a closure generating namespaced v5 UUIDs.
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @treturn function `factory`: a v5 UUID generator.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local fact = assert(uuid.factory_v5('e6ebd542-06ae-11e6-8e82-bba81706b27d'))
+        --
+        -- local u1 = fact('hello')
+        -- ---> 4850816f-1658-5890-8bfd-1ed14251f1f0
+        --
+        -- local u2 = fact('foobar')
+        -- ---> c9be99fc-326b-5066-bdba-dcd31a6d01ab
+        do
+            local ffi = require 'ffi'
+
+            local sha1_bin = ngx.sha1_bin
+            local C = ffi.C
+            local ffi_new = ffi.new
+            local ffi_str = ffi.string
+            local str_type = ffi.typeof('uint8_t[?]')
+            ffi.cdef [[
+            typedef unsigned char u_char;
+            u_char * ngx_hex_dump(u_char *dst, const u_char *src, size_t len);
+            ]]
+
+            local function bin_tohex(s)
+                local len = #s * 2
+                local buf = ffi_new(str_type, len)
+                C.ngx_hex_dump(buf, s, #s)
+                return ffi_str(buf, len)
+            end
+
+            local function v5_hash(binary, name)
+                local hash = bin_tohex(sha1_bin(binary..name))
+                return  hash,
+                tohex(bor(band(tonumber(sub(hash, 13, 14), 16), 0x0F), 0x50), 2),
+                tohex(bor(band(tonumber(sub(hash, 17, 18), 16), 0x3F), 0x80), 2)
+            end
+
+            function _M.factory_v5(namespace)
+                return factory(namespace, v5_hash)
+            end
+        end
+
+        --- Generate a v3 UUID.
+        -- v3 UUIDs are created from a namespace and a name (a UUID and a string).
+        -- The same name and namespace result in the same UUID. The same name and
+        -- different namespaces result in different UUIDs, and vice-versa.
+        -- The resulting UUID is derived using MD5 hashing.
+        --
+        -- This is a sugar function which instanciates a short-lived v3 UUID factory.
+        -- It is an expensive operation, and intensive generation using the same
+        -- namespaces should prefer allocating their own long-lived factory with
+        -- `factory_v3`.
+        --
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @param[type=string] name
+        -- @treturn string `uuid`: a v3 (namespaced) UUID.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local u = uuid.generate_v3('e6ebd542-06ae-11e6-8e82-bba81706b27d', 'hello')
+        -- ---> 3db7a435-8c56-359d-a563-1b69e6802c78
+        function _M.generate_v3(namespace, name)
+            local fact, err = _M.factory_v3(namespace)
+            if not fact then return nil, err end
+
+            return fact(name)
+        end
+
+        --- Generate a v5 UUID.
+        -- v5 UUIDs are created from a namespace and a name (a UUID and a string).
+        -- The same name and namespace result in the same UUID. The same name and
+        -- different namespaces result in different UUIDs, and vice-versa.
+        -- The resulting UUID is derived using SHA-1 hashing.
+        --
+        -- This is a sugar function which instanciates a short-lived v5 UUID factory.
+        -- It is an expensive operation, and intensive generation using the same
+        -- namespaces should prefer allocating their own long-lived factory with
+        -- `factory_v5`.
+        --
+        -- @param[type=string] namespace (must be a valid UUID according to `is_valid`)
+        -- @param[type=string] name
+        -- @treturn string `uuid`: a v5 (namespaced) UUID.
+        -- @treturn string `err`: a string describing an error
+        -- @usage
+        -- local uuid = require 'resty.jit-uuid'
+        --
+        -- local u = uuid.generate_v5('e6ebd542-06ae-11e6-8e82-bba81706b27d', 'hello')
+        -- ---> 4850816f-1658-5890-8bfd-1ed14251f1f0
+        function _M.generate_v5(namespace, name)
+            local fact, err = _M.factory_v5(namespace)
+            if not fact then return nil, err end
+
+            return fact(name)
+        end
+
+    else
+
+        function _M.factory_v3() error('v3 UUID generation only supported in ngx_lua', 2) end
+        function _M.generate_v3() error('v3 UUID generation only supported in ngx_lua', 2) end
+        function _M.factory_v5() error('v5 UUID generation only supported in ngx_lua', 2) end
+        function _M.generate_v5() error('v5 UUID generation only supported in ngx_lua', 2) end
+
+    end
+end
+
+return setmetatable(_M, {
+    __call = _M.generate_v4
+})

curve-snapshotcloneserver-nginx/app/src/init.lua

@@ -0,0 +1,21 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+require "resty.core"
+local checkups = require "resty.checkups.api"
+config = require "config"
+
+config.global.version = "v0.0.1"
+checkups.init(config)

curve-snapshotcloneserver-nginx/app/src/init_worker.lua

@@ -0,0 +1,24 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+local uuid     = require 'resty.uuid'
+local checkups = require "resty.checkups.api"
+
+local config = config
+
+checkups.prepare_checker(config)
+checkups.create_checker()
+
+uuid.seed()

curve-snapshotcloneserver-nginx/app/src/modules/httpipe.lua

@@ -0,0 +1,964 @@
+-- Copyright (C) Monkey Zhang (timebug), UPYUN Inc.
+
+
+local type = type
+local error = error
+local pairs = pairs
+local ipairs = ipairs
+local rawset = rawset
+local rawget = rawget
+local sub = string.sub
+local gsub = string.gsub
+local find = string.find
+local tostring = tostring
+local tonumber = tonumber
+local tcp = ngx.socket.tcp
+local match = string.match
+local upper = string.upper
+local lower = string.lower
+local concat = table.concat
+local insert = table.insert
+local format = string.format
+local setmetatable = setmetatable
+local ngx_re_match = ngx.re.match
+local encode_args = ngx.encode_args
+local ngx_req_socket = ngx.req.socket
+local ngx_req_get_headers = ngx.req.get_headers
+
+
+local _M = { _VERSION = "0.08" }
+
+--------------------------------------
+-- LOCAL CONSTANTS                  --
+--------------------------------------
+
+local mt = { __index = _M }
+
+local HTTP = {
+    [11] = " HTTP/1.1\r\n",
+    [10] = " HTTP/1.0\r\n"
+}
+
+local PORT = {
+    http = 80,
+    https = 443
+}
+
+local USER_AGENT = "Resty/HTTPipe-" .. _M._VERSION
+
+local STATE_NOT_READY = 0
+local STATE_BEGIN = 1
+local STATE_READING_HEADER = 2
+local STATE_READING_BODY = 3
+local STATE_EOF = 4
+
+local common_headers = {
+    "Cache-Control",
+    "Content-Length",
+    "Content-Type",
+    "Date",
+    "ETag",
+    "Expires",
+    "Host",
+    "Location",
+    "User-Agent"
+}
+
+for _, key in ipairs(common_headers) do
+    rawset(common_headers, key, key)
+    rawset(common_headers, lower(key), key)
+end
+
+local state_handlers
+
+--------------------------------------
+-- HTTP BASE FUNCTIONS              --
+--------------------------------------
+
+local function normalize_header(key)
+    local val = common_headers[key]
+    if val then
+        return val
+    end
+    key = lower(key)
+    val = common_headers[lower(key)]
+    if val then
+        return val
+    end
+
+    key = gsub(key, "^%l", upper)
+    key = gsub(key, "-%l", upper)
+    return key
+end
+
+
+local function req_header(self, opts)
+    self.method = upper(opts.method or "GET")
+
+    local req = {
+        self.method,
+        " "
+    }
+
+    local path = opts.path
+    if type(path) ~= "string" then
+        path = "/"
+    elseif sub(path, 1, 1) ~= "/" then
+        path = "/" .. path
+    end
+    insert(req, path)
+
+    if type(opts.query) == "table" then
+        opts.query = encode_args(opts.query)
+    end
+
+    if type(opts.query) == "string" then
+        insert(req, "?" .. opts.query)
+    end
+
+    insert(req, HTTP[opts.version])
+
+    opts.headers = opts.headers or {}
+
+    local headers = {}
+
+    for k, v in pairs(opts.headers) do
+        if opts.unnormalize_header then
+            headers[k] = v
+        else
+            headers[normalize_header(k)] = v
+        end
+    end
+
+    if type(opts.body) == "string" then
+        headers["Content-Length"] = #opts.body
+    elseif self.previous.content_length and
+    self.previous.content_length >= 0 then
+        headers["Content-Length"] = self.previous.content_length
+    end
+
+    if type(opts.body) == "function" and
+    not headers["Content-Length"] and not headers["Transfer-Encoding"] then
+        headers["Transfer-Encoding"] = "chunked"
+    end
+
+    if not headers["Host"] then
+        headers["Host"] = self.host
+    end
+
+    if not headers["User-Agent"] then
+        headers["User-Agent"] = USER_AGENT
+    end
+
+    if not headers["Accept"] then
+        headers["Accept"] = "*/*"
+    end
+
+    if opts.version == 10 and not headers["Connection"] then
+        headers["Connection"] = "keep-alive"
+    end
+
+    for key, values in pairs(headers) do
+        if type(values) ~= "table" then
+            values = { values }
+        end
+
+        key = tostring(key)
+        for _, value in pairs(values) do
+            insert(req, key .. ": " .. tostring(value) .. "\r\n")
+        end
+    end
+
+    insert(req, "\r\n")
+
+    return concat(req), headers
+end
+
+
+-- local scheme, host, port, path, args = unpack(_M:parse_uri(uri))
+function _M.parse_uri(self, uri)
+    local r = [[^(https?)://([^:/]+)(?::(\d+))?(.*)]]
+    local m, err = ngx_re_match(uri, r, "jo")
+    if not m then
+        return nil, err or "bad uri"
+    end
+
+    if not m[3] then m[3] = PORT[m[1]] end
+    if not m[4] then
+        m[4] = "/"
+    else
+        local raw = m[4]
+        local from = find(raw, "?")
+        if from then
+            m[4] = raw:sub(1, from - 1) -- path
+            m[5] = raw:sub(from + 1)    -- args
+        end
+    end
+
+    return m
+end
+
+
+local function init(self)
+    self.total_size = 0
+    self.state = STATE_NOT_READY
+    self.chunked = false
+    self.keepalive = true
+    self._eof = false
+    self.previous = {}
+    self.remaining = nil
+
+    return self
+end
+
+
+--------------------------------------
+-- HTTP PIPE FUNCTIONS              --
+--------------------------------------
+
+-- local hp, err = _M:new(chunk_size?, sock?)
+function _M.new(self, chunk_size, sock)
+    if not sock then
+        local s, err = tcp()
+        if not s then
+            return nil, err
+        end
+        sock = s
+    end
+
+    return setmetatable(init({
+        sock = sock,
+        chunk_size = chunk_size or 8192,
+    }), mt)
+end
+
+
+-- local ok, err = _M:set_timeout(time)
+function _M.set_timeout(self, time)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    return sock:settimeout(time)
+end
+
+
+-- local session, err = _M:ssl_handshake(self, ...)
+function _M.ssl_handshake(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    if not ngx.config
+        or not ngx.config.ngx_lua_version
+        or ngx.config.ngx_lua_version < 9011
+    then
+        error("ngx_lua 0.9.11+ required")
+    end
+
+    return sock:sslhandshake(...)
+end
+
+
+-- local ok, err = _M:connect(self, host, port, opts?)
+-- local ok, err = _M:connect("unix:/path/to/unix-domain.socket", opts?)
+function _M.connect(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    self.host = select(1, ...)
+    if sub(self.host, 1, 5) == "unix:" then
+        -- https://tools.ietf.org/html/rfc2616#section-14.23
+        -- A client MUST include a Host header field in all HTTP/1.1 request
+        -- messages . If the requested URI does not include an Internet host
+        -- name for the service being requested, then the Host header field MUST
+        -- be given with an empty value.
+        self.host = ""
+    end
+
+    self.port = select(2, ...)
+    if type(self.port) == "string" then
+        self.port = tonumber(self.port)
+    elseif type(self.port) ~= "number" then
+        self.port = nil
+    end
+
+    return sock:connect(...)
+end
+
+
+local function discard_line(self)
+    local read_line = self.read_line
+
+    local line, err = read_line()
+    if not line then
+        return nil, err
+    end
+
+    return 1
+end
+
+
+local function should_receive_body(method, code)
+    if method == "HEAD" then return nil end
+    if code == 204 or code == 304 then return nil end
+    if code >= 100 and code < 200 then return nil end
+    return true
+end
+
+
+local function read_body_part(self)
+    if not self.is_req_socket and
+    not should_receive_body(self.method, self.status_code) then
+        self.state = STATE_EOF
+        return 'body_end', nil
+    end
+
+    local sock = self.sock
+    local remaining = self.remaining
+    local chunk_size = self.chunk_size
+
+    if self.maxsize and remaining and remaining > self.maxsize then
+        return nil, nil, "exceeds maxsize"
+    end
+
+    if self.chunked == true and
+    (remaining == nil or remaining == 0) then
+        local read_line = self.read_line
+        local data, err = read_line()
+
+        if err then
+            return nil, nil, err
+        end
+
+        if data == "" then
+            data, err = read_line()
+            if err then
+                return nil, nil, err
+            end
+        end
+
+        if data then
+            if data == "0" then
+                local ok, err = discard_line(self)
+                if not ok then
+                    return nil, nil, err
+                end
+
+                self.state = STATE_EOF
+                return 'body_end', nil
+            else
+                local length = tonumber(data, 16)
+                remaining = length
+            end
+        end
+    end
+
+    if remaining == 0 then
+        self.state = STATE_EOF
+        return 'body_end', nil
+    end
+
+    if remaining ~= nil and remaining < chunk_size then
+        chunk_size = remaining
+    end
+
+    local chunk, err, partial = sock:receive(chunk_size)
+
+    local data = ""
+    if not err then
+        if chunk then
+            data = chunk
+        end
+    elseif err == "closed" then
+        self.state = STATE_EOF
+        if partial then
+            chunk_size = #partial
+            if remaining and remaining - chunk_size ~= 0 then
+                return nil, partial, err
+            end
+
+            data = partial
+        else
+            return 'body_end', nil
+        end
+    else
+        return nil, nil, err
+    end
+
+    if remaining ~= nil then
+        self.remaining = remaining - chunk_size
+        self.total_size = self.total_size + chunk_size
+
+        if self.maxsize and self.total_size > self.maxsize then
+            return nil, nil, "exceeds maxsize"
+        end
+    end
+
+    return 'body', data
+end
+
+
+local function read_header_part(self)
+    local read_line = self.read_line
+
+    local line, err = read_line()
+    if not line then
+        return nil, nil, err
+    end
+
+    if line == "" then
+        if self.chunked then
+            self.remaining = nil
+        end
+
+        self.state = STATE_READING_BODY
+        return 'header_end', nil
+    end
+
+    local m, err = ngx_re_match(line, [[^(.+?):\s*(.+)]], "jo")
+    if not m then
+        return 'header', line
+    end
+
+    local name, value = m[1], m[2]
+
+    local vname = lower(name)
+    if vname == "content-length" then
+        self.remaining = tonumber(value)
+    end
+
+    if vname == "transfer-encoding" and value ~= "identity" then
+        self.chunked = true
+    end
+
+    if vname == "connection" and value == "close" then
+        self.keepalive = value ~= "close"
+    end
+
+    return 'header', { normalize_header(name), value, line }
+end
+
+
+local function read_statusline(self)
+    local sock = self.sock
+    if self.read_line == nil then
+        local rl, err = sock:receiveuntil("\n")
+        if not rl then
+            return nil, nil, err
+        end
+        self.read_line = function()
+            --[[
+            The line terminator for message-header fields is the sequence CRLF.
+            However, we recommend that applications, when parsing such headers,
+            recognize a single LF as a line terminator and ignore the leading
+            CR.
+            REF: http://stackoverflow.com/questions/5757290/http-header-line-break-style
+            --]]
+            local data, err = rl()
+            if data and data:sub(-1) == "\r" then
+                data = data:sub(1, -2)
+            end
+
+            return data, err
+        end
+    end
+
+    local read_line = self.read_line
+
+    local line, err = read_line()
+    if not line then
+        return nil, nil, err
+    end
+
+    local version, status = match(line, "HTTP/(%d*%.%d*) (%d%d%d)")
+    if not version or not status then
+        -- return nil, nil, "not match statusline"
+        return nil, nil, line
+    end
+
+    version = tonumber(version) * 10
+    if version < 11 then
+        self.keepalive = false
+    end
+
+    self.status_code = tonumber(status)
+
+    if self.status_code == 100 then
+        local ok, err = discard_line(self)
+        if not ok then
+            return nil, nil, err
+        end
+
+        self.state = STATE_BEGIN
+    else
+        self.state = STATE_READING_HEADER
+    end
+
+    return 'statusline', status
+end
+
+
+-- local ok, err = _M:set_keepalive(...)
+function _M.set_keepalive(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    self._eof = true
+
+    if self.keepalive then
+        return sock:setkeepalive(...)
+    end
+
+    return sock:close()
+end
+
+
+-- local times, err = _M:get_reused_times()
+function _M.get_reused_times(self)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    return sock:getreusedtimes()
+end
+
+
+-- local ok, err = _M:close()
+function _M.close(self)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    self._eof = true
+
+    return sock:close()
+end
+
+
+local function read_eof(self)
+    if not self.is_req_socket then
+        self:set_keepalive()
+    end
+    return 'eof', nil
+end
+
+
+-- local typ, res, err = _M:read(chunk_size?)
+function _M.read(self, chunk_size)
+    local sock = self.sock
+    if not sock then
+        return nil, nil, "not initialized"
+    end
+
+    if chunk_size then
+        self.chunk_size = chunk_size
+    end
+
+    if self.state == STATE_NOT_READY then
+        return nil, nil, "not ready"
+    end
+
+    if self.read_timeout then
+        sock:settimeout(self.read_timeout)
+    end
+
+    local handler = state_handlers[self.state]
+    if handler then
+        return handler(self)
+    end
+
+    return nil, nil, "bad state: " .. self.state
+end
+
+
+-- local eof = _M:eof()
+function _M.eof(self)
+    return self._eof
+end
+
+
+state_handlers = {
+    read_statusline,
+    read_header_part,
+    read_body_part,
+    read_eof
+}
+
+
+-- local res, err = _M:read_response(callback?)
+function _M.read_response(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    local callback = ...
+    if type(callback) ~= "table" then
+        callback = {}
+    end
+
+    local status
+    local headers = {}
+    local chunks = {}
+
+    while not self._eof do
+        local typ, res, err = self:read()
+        if not typ then
+            return nil, err
+        end
+
+        if typ == 'statusline' then
+            status = tonumber(res)
+        end
+
+        if typ == 'header' then
+            if type(res) == "table" then
+                local key = res[1]
+                if headers[key] then
+                    if type(headers[key]) ~= "table" then
+                        headers[key] = { headers[key] }
+                    end
+                    insert(headers[key], tostring(res[2]))
+                else
+                    headers[key] = tostring(res[2])
+                end
+            end
+        end
+
+        if typ == 'header_end' then
+            if callback.header_filter then
+                local rc = callback.header_filter(status, headers)
+                if rc then break end
+            end
+        end
+
+        if typ == 'body' then
+            if callback.body_filter then
+                local rc = callback.body_filter(res)
+                if rc then break end
+            else
+                insert(chunks, res)
+            end
+        end
+
+        if typ == 'eof' then
+            break
+        end
+    end
+
+    return { status = status, headers = headers, body = concat(chunks) }
+end
+
+
+-- local ok, err = _M:send_request(opts?)
+function _M.send_request(self, opts)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    if opts.send_timeout then
+        sock:settimeout(opts.send_timeout)
+    end
+
+    if opts.read_timeout then
+        self.read_timeout = opts.read_timeout
+    end
+
+    if opts.version and not HTTP[opts.version] then
+        return nil, "unknown HTTP version"
+    else
+        opts.version = opts.version or 11
+    end
+
+    local req, headers = req_header(self, opts)
+    local bytes, err = sock:send(req)
+    if not bytes then
+        return nil, err
+    end
+
+    if type(opts.body) == "string" then
+        local bytes, err = sock:send(opts.body)
+        if not bytes then
+            return nil, err
+        end
+    elseif type(opts.body) == "function" then
+        local chunked = headers["Transfer-Encoding"] == "chunked"
+        repeat
+            local chunk, err = opts.body()
+            if chunk then
+                local size = #chunk
+                if chunked and size > 0 then
+                    chunk = concat({
+                            format("%X", size), "\r\n",
+                            chunk, "\r\n",
+                    })
+                end
+                local bytes, err = sock:send(chunk)
+                if not bytes then
+                    return nil, err
+                end
+            elseif err then
+                return nil, err
+            end
+        until not chunk
+        if chunked then
+            local bytes, err = sock:send("0\r\n\r\n")
+            if not bytes then
+                return nil ,err
+            end
+        end
+    end
+
+    self.maxsize = opts.maxsize
+    self.state = STATE_BEGIN
+
+    return 1
+end
+
+
+local function get_body_reader(self)
+    return function (chunk_size)
+        local typ, res, err = self:read(chunk_size)
+        if not typ then
+            return nil, err
+        end
+
+        if typ == 'body' then
+            return res
+        end
+
+        if typ == 'body_end' then
+            read_eof(self)
+        end
+
+        return
+    end
+end
+
+
+local function redirect(self, url, opts)
+    if sub(url, 1, 1) == "/" then
+        url = ("http://%s:%s%s"):format(gsub(self.host, [[:.+$]], ""), self.port, url)
+    end
+
+    local parsed_uri, _ = self:parse_uri(url)
+    if not parsed_uri then
+        return nil, "invalid redirect url"
+    end
+
+    local scheme, host, port, path, args = unpack(parsed_uri)
+    opts.path = path
+    opts.query = args
+
+    if scheme == "https" and host ~= opts.req_host then
+        opts.ssl_enable = true
+    else
+        opts.ssl_enable = false
+    end
+
+    if type(opts.is_valid_addr) == "function" then
+        local valid, format = opts.is_valid_addr(host)
+        if not valid then
+            return nil, "invalid redirect host"
+        end
+
+        -- 0 IP_FORMAT
+        -- 1 DOMAIN_FORMAT
+        if format == 1 then
+            opts.headers["Host"] = host
+        end
+    end
+
+    if opts.stream then
+        local res, err = self:read_response()
+        if not res then
+            return nil, err
+        end
+    end
+
+    init(self)
+
+    opts.redirect_follow_num = opts.redirect_follow_num - 1
+    if opts.redirect_follow_num <= 0 then
+        opts.allow_redirects = false
+    end
+
+    if host == opts.req_host then
+        host = gsub(self.host, [[:.+$]], "")
+        port = self.port
+    end
+
+    return self:request(host, port, opts)
+end
+
+
+-- local res, err = _M:request(opts?)
+-- local res, err = _M:request(host, port, opts?)
+-- local res, err = _M:request("unix:/path/to/unix-domain.socket", opts?)
+function _M.request(self, ...)
+    local sock = self.sock
+    if not sock then
+        return nil, "not initialized"
+    end
+
+    local arguments = {...}
+    local n = #arguments
+    if n > 3 then
+        return nil, "expecting 0, 1, 2, or 3 arguments, but seen " ..
+            tostring(n)
+    end
+
+    local opts = {}
+    if type(arguments[n]) == "table" then
+        opts = arguments[n]
+        arguments[n] = nil
+    end
+
+    if not opts.headers then
+        opts.headers = {}
+    end
+
+    if n > 0 and arguments[1] then
+        local rc, err = self:connect(unpack(arguments))
+        if not rc then
+            return nil, err
+        end
+
+        if self.port then
+            if opts.ssl_enable and self.port ~= 443 then
+                self.host = self.host .. ":" .. tostring(self.port)
+            elseif not opts.ssl_enable and self.port ~= 80 then
+                self.host = self.host .. ":" .. tostring(self.port)
+            end
+        end
+
+        if opts.ssl_enable then
+            local server_name = opts.headers["Host"] or self.host
+            if opts.ssl_server_name then
+                server_name = opts.ssl_server_name
+            end
+
+            local ssl_verify = true
+            if opts.ssl_verify == false then
+                ssl_verify = false
+            end
+
+            local ok, err = self:ssl_handshake(nil, server_name, ssl_verify)
+            if not ok then
+                return nil, err
+            end
+         end
+    end
+
+    local prev = self.previous.pipe
+    if self.previous.body_reader then
+        opts.body = self.previous.body_reader
+    end
+
+    local ok, err = self:send_request(opts)
+    if not ok then
+        if prev then prev:close() end
+        return nil, err
+    end
+
+    local res, err = self:read_response{
+        header_filter = function (status, headers)
+            return opts.stream
+        end
+    }
+
+    if res and opts.stream then
+        res.body_reader = get_body_reader(self)
+
+        local size = tonumber(res.headers["Content-Length"]) or -1 -- -1:chunked
+        local pipe, err = self:new(self.chunk_size)
+        if not pipe then
+            return nil, err
+        end
+
+        pipe.previous.pipe = self
+        pipe.previous.content_length = size
+        pipe.previous.body_reader = res.body_reader
+
+        res.pipe = pipe
+    end
+
+    if res and opts.allow_redirects
+        and (self.method == "GET" or self.method == "HEAD")
+        and (res.status == 301 or res.status == 302)
+    then
+        local url = res.headers["Location"]
+        if type(url) == "string" then
+            local _res, _err = redirect(self, url, opts)
+            if _res then
+                return _res, _err
+            end
+        end
+    end
+
+    return res, err
+end
+
+
+-- local res, err = _M:request_uri(uri, opts?)
+function _M.request_uri(self, uri, opts)
+    if not opts then
+        opts = {}
+    end
+
+    local parsed_uri, err = self:parse_uri(uri)
+    if not parsed_uri then
+        return nil, err
+    end
+
+    local scheme, host, port, path, args = unpack(parsed_uri)
+    if not opts.path then
+        opts.path = path
+    end
+
+    if not opts.query then
+        opts.query = args
+    end
+
+    if scheme == "https" then
+        opts.ssl_enable = true
+    end
+
+    return self:request(host, port, opts)
+end
+
+
+-- local reader, err = _M:get_client_body_reader(chunk_size?)
+function _M.get_client_body_reader(self, chunk_size)
+    local sock, err = ngx_req_socket()
+
+    if not sock then
+        return nil, err
+    end
+
+    local hp = self:new(chunk_size, sock)
+    local headers = ngx_req_get_headers()
+
+    hp.chunked = headers["Transfer-Encoding"] == "chunked"
+    if not hp.chunked then
+        hp.remaining = tonumber(headers["Content-Length"])
+    end
+
+    hp.state = STATE_READING_BODY
+    hp.is_req_socket = 1
+
+    return get_body_reader(hp)
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/modules/httproxy.lua

@@ -0,0 +1,392 @@
+-- Copyright (C) 2014 Jing Ye (yejingx), UPYUN Inc.
+
+local checkups = require "resty.checkups"
+local httpipe = require "modules.httpipe"
+local reqlimit = require "modules.reqlimit"
+local utils = require "modules.utils"
+
+local type = type
+local tostring = tostring
+local str_format = string.format
+local insert = table.insert
+local concat = table.concat
+local cjson = require "cjson.safe"
+
+
+local req_start_time = ngx.req.start_time
+local ngx_now = ngx.now
+local send_headers = ngx.send_headers
+
+
+local _M = { _VERSION = "0.10" }
+
+local function is_func(f)
+    return type(f) == "function"
+end
+
+local function set_uptime()
+    if ngx.var.uptime then
+        ngx.var.uptime = str_format("%.3f", ngx_now() - req_start_time())
+    end
+end
+
+
+local function set_xstate(errmsg)
+    if ngx.var.xstate then
+        ngx.var.xstate = tostring(errmsg)
+    end
+end
+
+
+local function set_upinfo(node, code, time, ctx)
+    local append_var = function(k, v)
+        if not v then
+            return
+        end
+
+        local value  = ngx.var[k]
+        if not value then
+            return
+        end
+
+        ngx.var[k] = value == '-' and v or value .. ", " .. v
+    end
+
+    if code and time and ctx.upstats == true and ctx.host and ctx.port then
+        local upstat = str_format("%s:%s/%s/%s", ctx.host, ctx.port, code, time)
+        append_var("upstats", upstat)
+    end
+
+    local srv = ctx.srv
+    if node and srv and srv.isp then
+        node = ("%s/%s"):format(srv.isp, node)
+    end
+
+    if node and srv and srv.domain then
+        node = ("%s-%s"):format(srv.domain, node)
+    end
+
+    append_var("upnode", node)
+    append_var("uptimes", time)
+    append_var("upcode", code)
+
+    if node and code and time and ngx.var.http_x_debug then
+        local upstat = str_format("%s/%s/%s", node, code, time)
+        append_var("uptrace", upstat)
+    end
+end
+
+
+function _M.close(hp)
+    if hp and not hp:eof() then
+        local ok, err = hp:close()
+        if not ok then -- warn: unread data in buffer
+            ngx.log(ngx.WARN, "failed to close: ", err)
+        end
+    end
+end
+
+
+local function prepare_opts(method, uri, source, opts)
+    local client_hp, err = httpipe:new()
+    if not client_hp then
+        return nil, err
+    end
+
+    local tm, send_tm, read_tm = checkups.get_ups_timeout(source)
+    local timeout = opts.timeout or tm or 5
+    local send_timeout = opts.send_timeout or send_tm or timeout
+    local read_timeout = opts.read_timeout or read_tm or timeout
+
+    local calc_timeout = opts.calc_timeout
+    if is_func(calc_timeout) then
+        timeout, send_timeout, read_timeout = calc_timeout(reqlimit.STATE_REQUEST, opts.slice)
+        if not timeout or not send_timeout or not read_timeout then
+            return nil, reqlimit.TIME_RUN_OUT
+        end
+    end
+
+    local method_with_request_body = {
+        --__FORWARD_GET_PAYLOAD__
+        GET = true, PUT = true, POST = true, DELETE = true, PATCH = true,
+        --__FORWARD_GET_PAYLOAD__
+    }
+    if not opts.body and method_with_request_body[method] then
+        opts.body = client_hp:get_client_body_reader(opts.chunk_size or 8192)
+    end
+
+    local req_opts = {
+        method = method, path = uri,
+        allow_redirects = opts.allow_redirects,
+        redirect_follow_num = opts.redirect_follow_num,
+        is_valid_addr = opts.is_valid_addr,
+        ssl_enable = opts.ssl_enable,
+        ssl_server_name = opts.ssl_server_name,
+        ssl_verify = opts.ssl_verify or false,
+        headers = opts.headers or {}, query = opts.query,
+        unnormalize_header = opts.unnormalize_header,
+        timeout = timeout * 1000,
+        send_timeout = send_timeout * 1000,
+        read_timeout = read_timeout * 1000,
+        body = opts.body,
+        maxsize = opts.max_body_size,
+        stream = true,
+        req_host = opts.req_host
+    }
+
+    return req_opts, nil
+end
+
+
+local function do_request(host, port, ctx, req_opts)
+    local res, err, hp
+    hp, err = httpipe:new()
+    if not hp then
+        return nil, err
+    end
+
+    ctx.hp = hp
+
+    local start = ngx.now()
+    hp:set_timeout(req_opts.timeout)
+
+    res, err = hp:request(host, port, req_opts)
+
+    ngx.log(ngx.INFO, "request ", host, ":", tostring(port), " ", cjson.encode(req_opts))
+
+    if not res then
+        ngx.log(ngx.WARN, "failed to request: ", host, ":", tostring(port), " ", err)
+    end
+
+    local uptime = ("%.3f"):format(ngx_now() - start)
+    local upcode = res and res.status or (err == "timeout" and err or "err")
+    local upnode = ("%s:%s"):format(host, port)
+
+    set_upinfo(upnode, upcode, uptime, ctx)
+
+    return res, err
+end
+
+
+local function capture(method, uri, source, opts)
+    opts = opts or {}
+    local req_opts, err = prepare_opts(method, uri, source, opts)
+    if not req_opts then
+        return nil, nil, err
+    end
+
+    local res
+    local calc_timeout = opts.calc_timeout
+
+    -- set_upinfo
+    local ctx = {
+        host = opts.host, port = opts.port,
+        srv = opts.srv, upstats = opts.upstats
+    }
+
+    if source then
+        res, err = checkups.ready_ok(source, do_request, {
+            args = { ctx, req_opts }, cluster_key = opts.cluster_key
+        })
+    else
+        res, err = do_request(opts.host, opts.port, ctx, req_opts)
+    end
+
+    local hp = ctx.hp
+    local failed_filter = opts.failed_capture_filter
+    local post_filter = opts.post_capture_filter
+
+    if not res then
+        if failed_filter then
+            failed_filter(source, err)
+        end
+        _M.close(hp)
+        return nil, nil, err
+    end
+
+    if post_filter then
+        post_filter(source, res)
+    end
+
+    res.headers["Transfer-Encoding"] = nil
+    res.headers["Connection"] = nil
+
+    local discard_abnormal_body = opts.discard_abnormal_body
+
+    local should_receive_body = true
+    if discard_abnormal_body and (method ~= "GET"
+        or (res.status ~= 200 and res.status ~= 206))
+    then
+        should_receive_body = false
+    end
+
+    if is_func(calc_timeout) then
+        local read_timeout = calc_timeout(reqlimit.STATE_READ)
+        if not read_timeout then
+            _M.close(hp)
+            return nil, nil, reqlimit.TIME_RUN_OUT
+        end
+
+        hp.read_timeout = read_timeout * 1000
+    end
+
+    if not should_receive_body or type(res.status) ~= "number" then
+        hp:read_response{ body_filter = function (chunk) end }
+        _M.close(hp)
+    end
+
+
+    return res, hp, nil
+end
+
+
+function _M.capture(method, uri, source, opts)
+    local res, hp, err = capture(method, uri, source, opts)
+
+    -- when POST failed(upstream 502/504/503/error)
+    -- discard req body explicitly
+    if (not res or (res and tonumber(res.status or 0) >= 500))
+       and method == "POST" and opts.body
+    then
+        local _chunk, _err
+        local reader = opts.body
+        repeat
+            _chunk, _err = reader()
+        until not _chunk
+    end
+
+    return res, hp, err
+end
+
+
+function _M.flush_response(hp, res, buffer_num, buffer_size, calc_timeout)
+    local buffer_num = buffer_num or 1
+    local buffer_size = buffer_size or 8192
+    local buffer = {}
+
+    local chunk, err
+    local reader = res.body_reader
+    repeat
+        if is_func(calc_timeout) then
+            hp.read_timeout = (calc_timeout(reqlimit.STATE_READ) or 0) * 1000
+        end
+
+        chunk, err = reader(buffer_size)
+
+        if err then
+            err = tostring(err) or "error"
+            if err == "timeout" then
+                err = "read timeout"
+            end
+            set_xstate(err)
+
+            if err ~= "closed" then
+                ngx.log(ngx.WARN, "failed to read response: ", err)
+            else
+                ngx.log(ngx.INFO, "failed to read response: ", err)
+            end
+
+            if err == "exceeds maxsize" then
+                _M.close(hp)
+                if not ngx.headers_sent then
+                    set_uptime()
+                    return ngx.exit(ngx.HTTP_FORBIDDEN)
+                end
+            end
+
+            if not ngx.headers_sent then
+                if ngx.ctx.slice then
+                    -- just return 423 for slice request when upstream
+                    -- prematurely closed connection
+                    ngx.status = 423
+                    set_uptime()
+                    return ngx.exit(423)
+                end
+
+                local ok, err = send_headers()
+                if not ok then
+                    ngx.log(ngx.WARN, "failed to send headers: ", err)
+                end
+            end
+
+            ngx.flush(true) -- flush response headers to client forcibly
+            set_uptime()
+            return ngx.exit(444)
+        end
+
+        if chunk then
+            if buffer_num == 1 then
+                -- The buffer_num in Shanks is always 1 so far. So never
+                -- insert it into table for avoiding the overhead table.concat.
+                ngx.print(chunk)
+                ngx.flush(true)
+            else
+                insert(buffer, chunk)
+                if #buffer >= buffer_num then
+                    ngx.print(concat(buffer))
+                    ngx.flush(true)
+                    buffer = {}
+                end
+            end
+        end
+    until not chunk
+
+    if #buffer > 0 then
+        ngx.print(concat(buffer))
+        ngx.flush(true)
+    end
+
+    --get stream size from httpipe.total_size
+    if ngx.var.upsize == "-" and ngx.var.from_upyun ~= "1" then
+        ngx.var.upsize = hp and hp.total_size or 0
+    end
+
+    set_uptime()
+end
+
+
+function _M.raise_capture_error(err, raise_error_filter)
+    set_uptime()
+
+    err = tostring(err) or "error"
+    ngx.log(ngx.WARN, "failed to capture: ", err)
+
+    set_xstate(err)
+
+    if type(raise_error_filter) == "function" then
+        return raise_error_filter(err)
+    end
+
+    if err == "timeout" then
+        return ngx.exit(ngx.HTTP_GATEWAY_TIMEOUT)
+    end
+
+    return ngx.exit(ngx.HTTP_SERVICE_UNAVAILABLE)
+end
+
+
+function _M.exec(method, uri, source, opts)
+    local res, hp, err = _M.capture(method, uri, source, opts)
+
+    if not res then
+        return _M.raise_capture_error(err)
+    end
+
+    if type(res.status) == "number" then
+        for k, v in pairs(res.headers) do
+            ngx.header[k] = v
+        end
+    end
+
+    ngx.status = res.status
+
+    if not hp:eof() then
+        return _M.flush_response(hp, res)
+    else
+        set_uptime()
+        return ngx.exit(res.status)
+    end
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/modules/reqlimit.lua

@@ -0,0 +1,83 @@
+-- Copyright(C) 2016 Jingli Chen (Wine93), UPYUN Inc.
+
+local math  = require "math"
+local utils = require "modules.utils"
+
+local ngx            = ngx
+local tonumber       = tonumber
+local floor          = math.floor
+local min            = math.min
+local ngx_now        = ngx.now
+local req_start_time = ngx.req.start_time
+local get_method     = ngx.get_method
+local is_tab         = utils.is_tab
+
+local _M = {
+    _VERSION = "0.01",
+
+    STATE_REQUEST = 0, -- connect, send, read header
+    STATE_READ    = 1, -- read body
+
+    TIME_RUN_OUT = 10,
+}
+local TIME_WEIGHT = { 0.1, 0.2, 0.7 }
+local TIME_LIMIT = 0.001
+
+
+function _M.register_callback(config)
+    if not is_tab(config) then
+        config = {}
+    end
+
+    local conn_timeout = config.cdn_timeout or 30
+    local send_timeout = config.cdn_send_timeout or 60
+    local read_timeout = config.cdn_read_timeout or 60
+    local start_time = req_start_time()
+    local bytes_sent = 0
+
+    local function calc_timeout(phase, slice)
+        local elapsed = ngx_now() - start_time
+        local downstream_timeout
+
+        if slice then
+            downstream_timeout = config.cdn_downstream_timeout_slice or 60
+        else
+            downstream_timeout = config.cdn_downstream_timeout or 60
+        end
+
+        local useable_time = downstream_timeout - elapsed
+
+        if phase == _M.STATE_REQUEST then
+            if useable_time <= 0 then
+                return
+            end
+
+            local timeout = { conn_timeout, send_timeout, read_timeout }
+            local time_alloc = {}
+            for i = 1, 3 do
+                time_alloc[i] = useable_time * TIME_WEIGHT[i]
+                time_alloc[i] = min(time_alloc[i], timeout[i])
+
+                if time_alloc[i] < TIME_LIMIT then
+                    time_alloc[i] = nil
+                end
+            end
+
+            return time_alloc[1], time_alloc[2], time_alloc[3]
+        end
+
+        -- phase == STATE_READ
+        local time4read = read_timeout
+        if bytes_sent == 0 then
+            bytes_sent = tonumber(ngx.var.bytes_sent) or 0
+            time4read = min(useable_time, read_timeout)
+        end
+
+        return time4read
+    end
+
+    return calc_timeout
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/modules/utils.lua

@@ -0,0 +1,251 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+-- Copyright (c) Netease, xiaojing tang
+
+local base     = require "resty.core.base"
+local debug    = require "debug"
+local argutils = require "resty.argutils"
+
+local _M = {}
+
+local next  = next
+local sub   = string.sub
+local gsub  = string.gsub
+local len   = string.len
+local find  = string.find
+local lower = string.lower
+
+local registry     = debug.getregistry()
+local ngx_match    = ngx.re.match
+local table_insert = table.insert
+local set_uri_args = ngx.req.set_uri_args
+local delete_args  = argutils.delete_args
+local escape_uri   = ngx.escape_uri
+local tab_insert   = table.insert
+local tab_concat   = table.concat
+local set_header   = ngx.req.set_header
+
+_M.is_str = function(s) return type(s) == "string" end
+_M.is_num = function(n) return type(n) == "number" end
+_M.is_tab = function(t) return type(t) == "table" end
+_M.is_func = function(f) return type(f) == "function" end
+_M.null = function(e) return e == nil or e == ngx.null end
+
+
+local inner_addr = {
+    ["10"]  = {0, 255},    -- 10.0.0.0-10.255.255.255
+    ["172"] = {16, 31},    -- 172.16.0.0-172.31.255.255
+    ["192"] = {168, 168},  -- 192.168.0.0-192.168.255.255
+}
+
+
+function _M.unormalize_header(s)
+    return gsub(lower(s), "-", "_")
+end
+
+
+function _M.is_empty_table(t)
+    return next( t ) == nil
+end
+
+
+function _M.escape_path(path)
+    return gsub(path, "([^/]+)", function (s) return escape_uri(s) end)
+end
+
+
+function _M.table_empty(t)
+    return next(t) == nil
+end
+
+
+function _M.table_in(t, key)
+    for k, v in ipairs(t) do
+        if key == v then
+            return true
+        end
+    end
+
+    return false
+end
+
+
+function _M.get_ip_port(host)
+    if not host then
+        return nil, nil
+    end
+
+    local pos = find(host, ":", 1)
+    if not pos then
+        return host, 80
+    end
+
+    local ip = sub(host, 1, pos - 1)
+    local port = tonumber(sub(host, pos + 1))
+
+    return ip, port
+end
+
+
+function _M.is_ip_format(host)
+    if not host == nil then
+        return false
+    end
+
+    local pos = find(host, ":", 1)
+    if pos then
+        host = sub(host, 1, pos - 1)
+    end
+
+    local m, err = ngx_match(host, [[^\d+\.\d+\.\d+\.\d+$]], "ijso")
+    if m and m[0] then
+        return true
+    else
+        return false
+    end
+end
+
+
+function _M.log_request_headers(msg)
+    local h = ngx.req.get_headers()
+    local t = {}
+    for k, v in pairs(h) do
+        tab_insert(t, ("%s=%s"):format(k, v))
+    end
+    ngx.log(ngx.ERR, msg, ", request headers:", tab_concat(t, " "))
+end
+
+
+function _M.gen_resource(bucket, object)
+    local resource = ""
+    if bucket and bucket ~= "" then
+        resource = resource .. "/" .. bucket
+    end
+
+    if object and object ~= "" then
+        resource = resource .. "/" .. object
+    end
+
+    return resource
+end
+
+
+function _M.split_string(s, p)
+    local res = {}
+    gsub(s, '[^'..p..']+', function(w) table_insert(res, w) end)
+    return res
+end
+
+
+function _M.is_inner_ip(ip)
+    local sub = _M.split_string(ip, ".")
+
+    if sub and #sub == 4 and sub[1] and sub[2] then
+        local range = inner_addr[sub[1]]
+        local int_sub2 = tonumber(sub[2]) or 0
+        if range and int_sub2 >= range[1] and int_sub2 <= range[2] then
+            return true
+        end
+    end
+
+    return false
+end
+
+
+function _M.set_isp(ip, special_ip)
+    -- from multi line, do not set isp
+    local isp = ngx.var.http_x_nos_isp
+    if isp then
+        set_header("x-nos-isp", nil)
+        set_header("x-inner-isp", isp)
+        return
+    end
+
+    if _M.is_inner_ip(ip) then
+        set_header("x-inner-isp", "inner")
+    elseif special_ip and special_ip[ip] then
+        set_header("x-inner-isp", "bp")
+    else
+        set_header("x-inner-isp", "outer")
+    end
+end
+
+
+function _M.delete_uri_args(del_args)
+    local args = ngx.var.args
+    if not args or not del_args then
+        return
+    end
+
+    local opts = { delete_bool_value = true }
+    for _, k in ipairs(del_args) do
+        args = delete_args(args, "^" .. k .. "$", ".*", opts)
+    end
+
+    set_uri_args(args)
+end
+
+
+function _M.table_dup(ori_tab)
+    if type(ori_tab) ~= "table" then
+        return ori_tab
+    end
+    local new_tab = {}
+    for k, v in pairs(ori_tab) do
+        if type(v) == "table" then
+            new_tab[k] = _M.table_dup(v)
+        else
+            new_tab[k] = v
+        end
+    end
+    return new_tab
+end
+
+
+-- stash the old ngx.ctx, create a new anchor.
+-- Note: stash_ngx_ctx and apply_ngx_ctx need to be called in pairs, otherwise
+-- the overhead memory leak will happens!
+function _M.stash_ngx_ctx()
+    local ctxs = registry.ngx_lua_ctx_tables
+    local ctx_ref = base.ref_in_table(ctxs, ngx.ctx)
+    ngx.var.ctx_ref = tostring(ctx_ref)
+end
+
+
+-- restore the old ngx.ctx with the anchor fetched from stash_ngx_ctx, replace
+-- current the ngx.ctx, you need to call it early after Nginx internal redirect
+-- happens.
+-- Note: stash_ngx_ctx and apply_ngx_ctx need to be called in pairs, otherwise
+-- the overhead memory leak will happens!
+function _M.apply_ngx_ctx()
+    local ctx_ref = tonumber(ngx.var.ctx_ref)
+    if not ctx_ref then
+        return
+    end
+
+    local ctxs = registry.ngx_lua_ctx_tables
+    local origin_ngx_ctx = ctxs[ctx_ref]
+    ngx.ctx = origin_ngx_ctx
+
+    --- FIXME unref the ctx_ref for avoiding memory leak
+    local FREE_LIST_REF = 0
+    ctxs[ctx_ref] = ctxs[FREE_LIST_REF]
+    ctxs[FREE_LIST_REF] = ctx_ref
+    ngx.var.ctx_ref = ""
+end
+
+
+return _M

curve-snapshotcloneserver-nginx/app/src/snapshot.lua

@@ -0,0 +1,18 @@
+--
+--  Copyright (c) 2020 NetEase Inc.
+--
+--  Licensed under the Apache License, Version 2.0 (the "License");
+--  you may not use this file except in compliance with the License.
+--  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+
+local httproxy = require "modules.httproxy"
+
+return httproxy.exec(ngx.var.request_method, ngx.var.request_uri, "snapshot");

curve-snapshotcloneserver-nginx/conf/mime.types

@@ -0,0 +1,94 @@
+types {
+    text/html                                        html htm shtml;
+    text/css                                         css;
+    text/xml                                         xml;
+    image/gif                                        gif;
+    image/jpeg                                       jpeg jpg;
+    application/javascript                           js;
+    application/atom+xml                             atom;
+    application/rss+xml                              rss;
+
+    text/mathml                                      mml;
+    text/plain                                       txt;
+    text/vnd.sun.j2me.app-descriptor                 jad;
+    text/vnd.wap.wml                                 wml;
+    text/x-component                                 htc;
+
+    image/png                                        png;
+    image/svg+xml                                    svg svgz;
+    image/tiff                                       tif tiff;
+    image/vnd.wap.wbmp                               wbmp;
+    image/webp                                       webp;
+    image/x-icon                                     ico;
+    image/x-jng                                      jng;
+    image/x-ms-bmp                                   bmp;
+
+    application/font-woff                            woff;
+    application/java-archive                         jar war ear;
+    application/json                                 json;
+    application/mac-binhex40                         hqx;
+    application/msword                               doc;
+    application/pdf                                  pdf;
+    application/postscript                           ps eps ai;
+    application/rtf                                  rtf;
+    application/vnd.apple.mpegurl                    m3u8;
+    application/vnd.google-earth.kml+xml             kml;
+    application/vnd.google-earth.kmz                 kmz;
+    application/vnd.ms-excel                         xls;
+    application/vnd.ms-fontobject                    eot;
+    application/vnd.ms-powerpoint                    ppt;
+    application/vnd.oasis.opendocument.graphics      odg;
+    application/vnd.oasis.opendocument.presentation  odp;
+    application/vnd.oasis.opendocument.spreadsheet   ods;
+    application/vnd.oasis.opendocument.text          odt;
+    application/vnd.openxmlformats-officedocument.presentationml.presentation
+                                                     pptx;
+    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+                                                     xlsx;
+    application/vnd.openxmlformats-officedocument.wordprocessingml.document
+                                                     docx;
+    application/vnd.wap.wmlc                         wmlc;
+    application/x-7z-compressed                      7z;
+    application/x-cocoa                              cco;
+    application/x-java-archive-diff                  jardiff;
+    application/x-java-jnlp-file                     jnlp;
+    application/x-makeself                           run;
+    application/x-perl                               pl pm;
+    application/x-pilot                              prc pdb;
+    application/x-rar-compressed                     rar;
+    application/x-redhat-package-manager             rpm;
+    application/x-sea                                sea;
+    application/x-shockwave-flash                    swf;
+    application/x-stuffit                            sit;
+    application/x-tcl                                tcl tk;
+    application/x-x509-ca-cert                       der pem crt;
+    application/x-xpinstall                          xpi;
+    application/xhtml+xml                            xhtml;
+    application/xspf+xml                             xspf;
+    application/zip                                  zip;
+
+    application/octet-stream                         bin exe dll;
+    application/octet-stream                         deb;
+    application/octet-stream                         dmg;
+    application/octet-stream                         iso img;
+    application/octet-stream                         msi msp msm;
+
+    audio/midi                                       mid midi kar;
+    audio/mpeg                                       mp3;
+    audio/ogg                                        ogg;
+    audio/x-m4a                                      m4a;
+    audio/x-realaudio                                ra;
+
+    video/3gpp                                       3gpp 3gp;
+    video/mp2t                                       ts;
+    video/mp4                                        mp4;
+    video/mpeg                                       mpeg mpg;
+    video/quicktime                                  mov;
+    video/webm                                       webm;
+    video/x-flv                                      flv;
+    video/x-m4v                                      m4v;
+    video/x-mng                                      mng;
+    video/x-ms-asf                                   asx asf;
+    video/x-ms-wmv                                   wmv;
+    video/x-msvideo                                  avi;
+}

curve-snapshotcloneserver-nginx/conf/nginx.conf

@@ -0,0 +1,83 @@
+# -*- mode: nginx -*-
+# vim: set expandtab tabstop=4 shiftwidth=4:
+
+#user  nobody;
+worker_processes  5;
+error_log  logs/error.log  info;
+pid        logs/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for", "$http_range", "$request_time" '
+                      '"$upstream_addr" "$upnode" "$upcode" ""$uptime" ';
+
+    access_log  logs/access.log  main;
+
+    sendfile on;
+    keepalive_timeout  65;
+
+    send_timeout 60;
+    proxy_read_timeout 60;
+    proxy_send_timeout 60;
+    proxy_connect_timeout 1;
+    proxy_max_temp_file_size 0m;
+
+    client_header_buffer_size 4k;
+    large_client_header_buffers 8 16k;
+    client_body_buffer_size 128k;
+    client_max_body_size 200m;
+
+    port_in_redirect off;
+    open_log_file_cache max=2048 inactive=60s min_uses=2 valid=15m;
+
+    resolver 119.29.29.29 223.5.5.5 valid=1s;
+    resolver_timeout 5s;
+
+    lua_max_running_timers 512;
+    lua_max_pending_timers 1024;
+
+    lua_package_path "$prefix/lualib/?.lua;$prefix/app/lib/?.lua;$prefix/app/etc/?.lua;$prefix/app/src/?.lua;;";
+    lua_package_cpath "$prefix/lualib/?.so;/$prefix/app/lib/?.so;;";
+
+    lua_check_client_abort on;
+    lua_socket_log_errors off;
+    lua_http10_buffering off;
+
+    lua_shared_dict state 10m;
+    lua_shared_dict mutex 1m;
+    lua_shared_dict locks 1m;
+    lua_shared_dict config 1m;
+    lua_shared_dict cache 1m;
+
+    init_by_lua_file app/src/init.lua;
+    init_worker_by_lua_file app/src/init_worker.lua;
+
+
+    server {
+        listen 80 default_server;
+        server_name _;
+
+        set $upnode "-";
+        set $upcode "-";
+        set $upsize "-";
+        set $uptime "-";
+        set $uptimes "-";
+        set $upstats "-";
+        set $xstate "-";
+
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+
+        location / {
+            content_by_lua_file app/src/snapshot.lua;
+        }
+    }
+}

curve-snapshotcloneserver-nginx/docker/Dockerfile

@@ -0,0 +1,62 @@
+# Dockerfile - Debian 10 Buster - DEB version
+# https://github.com/openresty/docker-openresty
+
+ARG RESTY_IMAGE_BASE="debian"
+ARG RESTY_IMAGE_TAG="buster-slim"
+
+FROM ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG}
+
+LABEL maintainer="Evan Wies <evan@neomantra.net>"
+
+# RESTY_DEB_FLAVOR build argument is used to select other
+# OpenResty Debian package variants.
+# For example: "-debug" or "-valgrind"
+ARG RESTY_DEB_FLAVOR=""
+ARG RESTY_DEB_VERSION="=1.15.8.3-1~buster1"
+ARG RESTY_IMAGE_BASE="debian"
+ARG RESTY_IMAGE_TAG="buster-slim"
+
+LABEL resty_image_base="${RESTY_IMAGE_BASE}"
+LABEL resty_image_tag="${RESTY_IMAGE_TAG}"
+LABEL resty_deb_flavor="${RESTY_DEB_FLAVOR}"
+LABEL resty_deb_version="${RESTY_DEB_VERSION}"
+
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        ca-certificates \
+        gettext-base \
+        gnupg2 \
+        lsb-release \
+        software-properties-common \
+        wget \
+    && wget -qO /tmp/pubkey.gpg https://openresty.org/package/pubkey.gpg \
+    && DEBIAN_FRONTEND=noninteractive apt-key add /tmp/pubkey.gpg \
+    && rm /tmp/pubkey.gpg \
+    && DEBIAN_FRONTEND=noninteractive add-apt-repository -y "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" \
+    && DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge \
+        gnupg2 \
+        lsb-release \
+        software-properties-common \
+        wget \
+    && DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        openresty${RESTY_DEB_FLAVOR}${RESTY_DEB_VERSION} \
+    && DEBIAN_FRONTEND=noninteractive apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /var/run/openresty \
+    && ln -sf /dev/stdout /usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/logs/access.log \
+    && ln -sf /dev/stderr /usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/logs/error.log
+
+# Add additional binaries into PATH for convenience
+ENV PATH="$PATH:/usr/local/openresty${RESTY_DEB_FLAVOR}/luajit/bin:/usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/sbin:/usr/local/openresty${RESTY_DEB_FLAVOR}/bin"
+
+# Copy nginx configuration files
+COPY nginx.conf /usr/local/openresty${RESTY_DEB_FLAVOR}/nginx/conf/nginx.conf
+COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf
+
+CMD ["/usr/bin/openresty", "-g", "daemon off;"]
+
+# Use SIGQUIT instead of default SIGTERM to cleanly drain requests
+# See https://github.com/openresty/docker-openresty/blob/master/README.md#tips--pitfalls
+STOPSIGNAL SIGQUIT

curvefs/README.md

@@ -1,18 +0,0 @@
-CURVEFS
-===
-
-Curve FileSystem
-
-Quick Start
-===
-
-```
-$ make build dep=1
-```
-
-OR
-
-```
-$ make dep
-$ make build
-```

curvefs/conf/client.conf

@@ -1,273 +0,0 @@
-##### mdsOpt
-# RPC total retry time with MDS
-mdsOpt.mdsMaxRetryMS=16000
-# The maximum timeout of RPC communicating with MDS.
-# The timeout of exponential backoff cannot exceed this value
-mdsOpt.rpcRetryOpt.maxRPCTimeoutMS=2000
-# RPC timeout for once communication with MDS
-mdsOpt.rpcRetryOpt.rpcTimeoutMs=500
-# RPC with mds needs to sleep for a period of time before each retry
-mdsOpt.rpcRetryOpt.rpcRetryIntervalUS=50000
-# Switch if the number of consecutive retries on the current MDS exceeds the limit.
-# The number of failures includes the number of timeout retries
-mdsOpt.rpcRetryOpt.maxFailedTimesBeforeChangeAddr=2
-# The normal retry times for trigger wait strategy
-mdsOpt.rpcRetryOpt.normalRetryTimesBeforeTriggerWait=3
-# Sleep interval for wait
-mdsOpt.rpcRetryOpt.waitSleepMs=1000
-mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702  # __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__
-
-
-#
-# lease options
-#
-# client and mds lease time, default is 20s
-mds.leaseTimesUs=20000000
-mds.refreshTimesPerLease=5
-
-#### metaCacheOpt
-# Gets the number of retries for the leader
-metaCacheOpt.metacacheGetLeaderRetry=3
-# Need to sleep for a period of time before each get leader retry
-metaCacheOpt.metacacheRPCRetryIntervalUS=100000
-# RPC timeout of get leader
-metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
-
-#### executorOpt
-# executorOpt rpc with metaserver
-# rpc retry times with metaserver
-executorOpt.maxRetry=4294967295
-# internal rpc retry times with metaserver
-executorOpt.maxInternalRetry=3
-# Retry sleep time between failed RPCs
-executorOpt.retryIntervalUS=100000
-# RPC timeout for communicating with metaserver
-executorOpt.rpcTimeoutMS=1000
-# RPC stream idle timeout
-executorOpt.rpcStreamIdleTimeoutMS=500
-# The maximum timeout RPC time of the retry request.
-# The timeout time will follow the exponential backoff policy.
-# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
-executorOpt.maxRPCTimeoutMS=8000
-# Maximum sleep time between retry requests.
-# when the network is congested or the metaserver is overloaded,
-# it is necessary to increase the sleep time
-executorOpt.maxRetrySleepIntervalUS=8000000
-executorOpt.minRetryTimesForceTimeoutBackoff=5
-executorOpt.maxRetryTimesBeforeConsiderSuspend=20
-# batch limit of get inode attr and xattr
-executorOpt.batchInodeAttrLimit=10000
-
-#### spaceserver
-spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceServer.rpcTimeoutMs=1000
-
-#### bdev
-# curve client's config file
-bdev.confPath=/etc/curve/client.conf
-
-#### extentManager
-extentManager.preAllocSize=65536
-
-#### brpc
-# close socket after defer.close.second
-rpc.defer.close.second=1
-# rpc health check interval in second, 0 or negative value means disable health check
-rpc.healthCheckIntervalSec=0
-
-#### fuseClient
-# TODO(xuchaojie): add unit
-fuseClient.listDentryLimit=65536
-fuseClient.downloadMaxRetryTimes=3
-
-### kvcache opt
-fuseClient.supportKVcache=false
-fuseClient.setThreadPool=4
-fuseClient.getThreadPool=4
-
-# you shoudle enable it when mount one filesystem to multi mountpoints,
-# it gurantee the consistent of file after rename, otherwise you should
-# disable it for performance.
-fuseClient.enableMultiMountPointRename=true
-# splice will bring higher performance in some cases
-# but there might be a kernel issue that will cause kernel panic when enabling it
-# see https://lore.kernel.org/all/CAAmZXrsGg2xsP1CK+cbuEMumtrqdvD-NKnWzhNcvn71RV3c1yw@mail.gmail.com/
-# until this issue has been fixed, splice should be disabled
-fuseClient.enableSplice=false
-# thread number of listDentry when get summary xattr
-fuseClient.listDentryThreads=10
-# disable xattr on one mountpoint can fast 'ls -l'
-fuseClient.disableXattr=false
-# default data（s3ChunkInfo/volumeExtent） size in inode, if exceed will eliminate and try to get the merged one
-fuseClient.maxDataSize=1024
-# default refresh data interval 30s
-fuseClient.refreshDataIntervalSec=30
-fuseClient.warmupThreadsNum=10
-
-# the write throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgWriteBytes=0
-# the write burst bps of fuseClient, default no limit
-fuseClient.throttle.burstWriteBytes=0
-# the times that write burst bps can continue, default 180s
-fuseClient.throttle.burstWriteBytesSecs=180
-
-# the write throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgWriteIops=0
-# the write burst iops of fuseClient, default no limit
-fuseClient.throttle.burstWriteIops=0
-# the times that write burst Iops can continue, default 180s
-fuseClient.throttle.burstWriteIopsSecs=180
-
-# the read throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgReadBytes=0
-# the read burst bps of fuseClient, default no limit
-fuseClient.throttle.burstReadBytes=0
-# the times that read burst bps can continue, default 180s
-fuseClient.throttle.burstReadBytesSecs=180
-
-# the read throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgReadIops=0
-# the read burst Iops of fuseClient, default no limit
-fuseClient.throttle.burstReadIops=0
-# the times that read burst Iops can continue, default 180s
-fuseClient.throttle.burstReadIopsSecs=180
-
-#### filesystem metadata
-# {
-# fs.disableXattr:
-#   if you want to get better metadata performance,
-#   you can mount fs with |fs.disableXattr| is true
-#
-# fs.lookupCache.negativeTimeoutSec:
-#   entry which not found will be cached if |timeout| > 0
-fs.cto=true
-fs.maxNameLength=255
-fs.disableXattr=false
-fs.accessLogging=true
-fs.kernelCache.attrTimeoutSec=3600
-fs.kernelCache.dirAttrTimeoutSec=3600
-fs.kernelCache.entryTimeoutSec=3600
-fs.kernelCache.dirEntryTimeoutSec=3600
-fs.lookupCache.negativeTimeoutSec=0
-fs.lookupCache.minUses=1
-fs.lookupCache.lruSize=100000
-fs.dirCache.lruSize=5000000
-fs.openFile.lruSize=65536
-fs.attrWatcher.lruSize=5000000
-fs.rpc.listDentryLimit=65536
-fs.deferSync.delay=3
-fs.deferSync.deferDirMtime=false
-# }
-
-#### volume
-volume.bigFileSize=1048576
-volume.volBlockSize=4096
-volume.fsBlockSize=4096
-
-# allocator type, supported {bitmap}
-volume.allocator.type=bitmap
-
-## for bitmap allocator
-# size of each bit, default is 4MiB
-volume.bitmapAllocator.sizePerBit=4194304
-
-# small allocation proportion [0-1]
-volume.bitmapAllocator.smallAllocProportion=0.2
-
-# number of block groups that allocated once
-volume.blockGroup.allocateOnce=4
-
-#### s3
-# this is for test. if s3.fakeS3=true, all data will be discarded
-s3.fakeS3=false
-s3.pageSize=65536
-# prefetch blocks that disk cache use
-s3.prefetchBlocks=1
-# prefetch threads
-s3.prefetchExecQueueNum=1
-# start sleep when mem cache use ratio is greater than nearfullRatio,
-# sleep time increase follow with mem cache use ratio, baseSleepUs is baseline.
-s3.nearfullRatio=70
-s3.baseSleepUs=500
-
-# background thread schedule time
-s3.threadScheduleInterval=3
-# data cache flush wait time
-s3.cacheFlushIntervalSec=5
-# write cache < 8,388,608 (8MB) is not allowed
-s3.writeCacheMaxByte=838860800
-s3.readCacheMaxByte=209715200
-# file cache read thread num
-s3.readCacheThreads=5
-# http = 0, https = 1
-s3.http_scheme=0
-s3.verify_SSL=False
-s3.region=us-east-1
-s3.maxConnections=500
-s3.connectTimeout=60000
-s3.requestTimeout=10000
-# Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
-s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.asyncThreadNum=500
-# limit all inflight async requests' bytes, |0| means not limited
-s3.maxAsyncRequestInflightBytes=104857600
-s3.chunkFlushThreads=5
-# throttle
-s3.throttle.iopsTotalLimit=0
-s3.throttle.iopsReadLimit=0
-s3.throttle.iopsWriteLimit=0
-s3.throttle.bpsTotalMB=0
-s3.throttle.bpsReadMB=0
-s3.throttle.bpsWriteMB=0
-s3.useVirtualAddressing=false
-# The interval between read failures and retries will become larger and larger,
-# and when the max is reached, retry will be performed at a fixed time.
-s3.maxReadRetryIntervalMs = 1000
-# retry interval
-s3.readRetryIntervalMs = 100
-# TODO(hongsong): limit bytes、iops/bps
-#### disk cache options
-# 0:not enable disk cache
-# 1:readonly
-# 2:read/write
-diskCache.diskCacheType=2  # __ANSIBLE_TEMPLATE__ {{ client_disk_cache_type | default('2') }} __ANSIBLE_TEMPLATE__
-# the file system writes files use flush or not
-diskCache.forceFlush=true
-# the interval of check to trim disk cache
-diskCache.trimCheckIntervalSec=5
-# the interval of check to trim load file to s3
-diskCache.asyncLoadPeriodMs=5
-# start trim file when disk cache use ratio is Greater than fullRatio,
-# util less than safeRatio
-diskCache.fullRatio=90
-diskCache.safeRatio=70
-diskCache.threads=5
-# the max size disk cache can use
-diskCache.maxUsableSpaceBytes=107374182400
-# the max files that can cache
-diskCache.maxFileNums=1000000
-# the max time system command can run
-diskCache.cmdTimeoutSec=300
-# directory of disk cache
-diskCache.cacheDir=/mnt/curvefs_cache  # __CURVEADM_TEMPLATE__ /curvefs/client/data/cache __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /mnt/curvefs_disk_cache/{{ 99999999 | random | to_uuid | upper }} __ANSIBLE_TEMPLATE__
-
-# the write throttle bps of disk cache, default no limit
-diskCache.avgFlushBytes=0
-# the write burst bps of disk cache, default no limit
-diskCache.burstFlushBytes=0
-# the times that write burst bps can continue, default 180s
-diskCache.burstSecs=180
-# the write throttle iops of disk cache, default no limit
-diskCache.avgFlushIops=0
-# the read throttle bps of disk cache, default no limit
-diskCache.avgReadFileBytes=0
-# the read throttle iops of disk cache, default no limit
-diskCache.avgReadFileIops=0
-
-#### common
-client.common.logDir=/data/logs/curvefs  # __CURVEADM_TEMPLATE__ /curvefs/client/logs __CURVEADM_TEMPLATE__
-# we have loglevel: {0,3,6,9}
-# as the number increases, it becomes more and more detailed
-client.loglevel=0
-client.dummyServer.startPort=9000

curvefs/conf/etcd.conf

@@ -1,113 +0,0 @@
-# This is the configuration file for the etcd server.
-
-# Human-readable name for this member.
-name:  # __CURVEADM_TEMPLATE__ etcd${service_host_sequence}${service_replica_sequence} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ inventory_hostname }} __ANSIBLE_TEMPLATE__
-
-# Path to the data directory.
-data-dir:  # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_etcd_data_dir }} __ANSIBLE_TEMPLATE__
-
-# Path to the dedicated wal directory.
-wal-dir:  # __CURVEADM_TEMPLATE__ ${prefix}/data/wal __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_etcd_wal_dir }} __ANSIBLE_TEMPLATE__
-
-# Number of committed transactions to trigger a snapshot to disk.
-snapshot-count: 10000
-
-# Time (in milliseconds) of a heartbeat interval.
-heartbeat-interval: 100
-
-# Time (in milliseconds) for an election to timeout.
-election-timeout: 1000
-quota-backend-bytes: 0
-
-# List of comma separated URLs to listen on for peer traffic.
-listen-peer-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_peer_port }} __ANSIBLE_TEMPLATE__
-
-# List of comma separated URLs to listen on for client traffic.
-listen-client-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_client_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_client_port }} __ANSIBLE_TEMPLATE__
-
-# Maximum number of snapshot files to retain (0 is unlimited).
-max-snapshots: 5
-
-# Maximum number of wal files to retain (0 is unlimited).
-max-wals: 5
-
-# Comma-separated white list of origins for CORS (cross-origin resource sharing).
-cors:
-
-# List of this member's peer URLs to advertise to the rest of the cluster.
-# The URLs needed to be a comma-separated list.
-initial-advertise-peer-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_peer_port }} __ANSIBLE_TEMPLATE__
-
-# List of this member's client URLs to advertise to the public.
-# The URLs needed to be a comma-separated list.
-advertise-client-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_client_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ http://{{ ansible_ssh_host }}:{{ curvefs_etcd_listen_client_port }} __ANSIBLE_TEMPLATE__
-
-# Discovery URL used to bootstrap the cluster.
-discovery:
-
-# Valid values include 'exit', 'proxy'
-discovery-fallback: proxy
-
-# HTTP proxy to use for traffic to discovery service.
-discovery-proxy:
-
-# DNS domain used to bootstrap initial cluster.
-discovery-srv:
-
-# Initial cluster configuration for bootstrapping.
-initial-cluster:  # __CURVEADM_TEMPLATE__ ${cluster_etcd_http_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_etcd_peer(hostvars) }} __ANSIBLE_TEMPLATE__
-
-# Initial cluster token for the etcd cluster during bootstrap.
-initial-cluster-token: etcd-cluster
-
-# Initial cluster state ('new' or 'existing').
-initial-cluster-state: new
-
-# Reject reconfiguration requests that would cause quorum loss.
-strict-reconfig-check: False
-
-# Accept etcd V2 client requests
-enable-v2: True
-
-# Enable runtime profiling data via HTTP server
-enable-pprof: True
-
-# Valid values include 'on', 'readonly', 'off'
-proxy: 'off'
-
-# Time (in milliseconds) an endpoint will be held in a failed state.
-proxy-failure-wait: 5000
-
-# Time (in milliseconds) of the endpoints refresh interval.
-proxy-refresh-interval: 30000
-
-# Time (in milliseconds) for a dial to timeout.
-proxy-dial-timeout: 1000
-
-# Time (in milliseconds) for a write to timeout.
-proxy-write-timeout: 5000
-
-# Time (in milliseconds) for a read to timeout.
-proxy-read-timeout: 0
-
-# Enable debug-level logging for etcd.
-debug: False
-
-logger: zap
-
-# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
-log-outputs: [stderr]
-
-# Force to create a new one member cluster.
-force-new-cluster: False
-
-auto-compaction-mode: periodic
-auto-compaction-retention: "1"
-
-# Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
-metrics: extensive
-
-# Enable to run an additional Raft election phase.
-pre-vote: True
-
-enable-grpc-gateway: True

curvefs/conf/mds.conf

@@ -1,164 +0,0 @@
-#
-# mds options
-#
-mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service_port} __CURVEADM_TEMPLATE__   __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_host }}:{{ curvefs_mds_listen_port }} __ANSIBLE_TEMPLATE__
-# dummy server port
-mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
-mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
-mds.loglevel=0
-# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-mds.server.idleTimeoutSec=-1
-
-#
-# space options
-#
-space.addr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-space.rpcTimeoutMs=500
-
-#
-# metaserver options
-#
-metaserver.addr=127.0.0.1:6701  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.metaserver | join_peer(hostvars, "metaserver_listen_port") }} __ANSIBLE_TEMPLATE__
-metaserver.rpcTimeoutMs=5000
-metaserver.rpcRertyTimes=3
-metaserver.rpcRetryIntervalUs=1000000
-
-#
-# etcd options
-#
-# etcd listen address
-etcd.endpoint=127.0.0.1:2379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_peer(hostvars, "etcd_listen_client_port") }} __ANSIBLE_TEMPLATE__
-# timeout for establishing a connection
-etcd.dailtimeoutMs=5000
-# timeout for the operation
-etcd.operation.timeoutMs=5000
-# number of times a failed operation can be retried
-etcd.retry.times=3
-
-#
-# leader election options
-#
-leader.sessionInterSec=5
-leader.electionTimeoutMs=0
-
-#
-# topology config
-#
-# time interval flush data to db
-mds.topology.TopologyUpdateToRepoSec=60
-# max partition number in copyset 2^7
-mds.topology.MaxPartitionNumberInCopyset=128
-# inode number in each partition 2^20 [0, 2^20-1]
-mds.topology.IdNumberInPartition=1048576
-# default create partition number 12
-mds.topology.CreatePartitionNumber=12
-# max copyset num in metaserver
-mds.topology.MaxCopysetNumInMetaserver=100
-# Topology update metric interval
-mds.topology.UpdateMetricIntervalSec=60
-
-#
-# heartbeat config
-#
-# heartbeat interval between metaserver and mds
-mds.heartbeat.intervalMs=10000
-# heartbeat miss time between metaserver and mds
-mds.heartbeat.misstimeoutMs=30000
-# heartbeat offline time between metaserver and mds
-mds.heartbeat.offlinetimeoutMs=1800000
-# After mds is started, a certain time delay starts to guide the metaserver to delete data
-# default is 20 min
-mds.heartbeat.clean_follower_afterMs=1200000
-
-#
-# schedule config
-#
-# recoverScheduler switch
-mds.enable.recover.scheduler=true
-# copysetScheduler switch
-mds.enable.copyset.scheduler=true
-# leaderScheduler switch
-mds.enable.leader.scheduler=true
-# RecoverScheduler round interval, the unit is second
-mds.recover.scheduler.intervalSec=5
-# copysetScheduler round interval, the unit is second
-mds.copyset.scheduler.intervalSec=5
-# leaderScheduler round interval, the unit is second
-mds.leader.scheduler.intervalSec=5
-# the percentage difference for copysetScheduler to
-# determine whether resource balancing is required
-# based on the difference in resource usage percent, default is 15%
-mds.copyset.scheduler.balanceRatioPercent=15
-# Concurrency of operator on each metaserver
-mds.schduler.operator.concurrent=1
-# transfer leader timeout, after the timeout, mds removes the operator from the memory
-mds.schduler.transfer.limitSec=60
-# remove a replica timeout, after the timeout, mds removes the operator from the memory
-mds.scheduler.remove.limitSec=300
-# add a replica timeout, after the timeout, mds removes the operator from the memory
-mds.scheduler.add.limitSec=1800
-# change the replica timeout, after the timeout, mds removes the operator from the memory
-mds.scheduler.change.limitSec=1800
-# metaserver can be used as target leader only after starting coolingTimeSec_, the unit is second
-mds.scheduler.metaserver.cooling.timeSec=1800
-
-#
-# fsmananger config
-#
-# the backend thread check whether fs is able to delete,
-# check partition of deleting fs is deleting
-mds.fsmanager.backEndThreadRunInterSec=10
-# number of threads that load space info of volume
-mds.fsmanager.reloadSpaceConcurrency=10
-# the client timeout is 20s default, umount fs if timeout
-mds.fsmanager.client.timeoutSec=20
-
-#### s3
-# TODO(huyao): use more meaningfull name
-# http = 0, https = 1
-s3.http_scheme=0
-s3.verify_SSL=False
-s3.region=us-east-1
-s3.maxConnections=32
-s3.connectTimeout=60000
-s3.requestTimeout=10000
-# Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.logLevel=4
-s3.logPrefix=/data/logs/curvefs/aws_  # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.asyncThreadNum=30
-# limit all inflight async requests' bytes, |0| means not limited
-s3.maxAsyncRequestInflightBytes=104857600
-# throttle
-s3.throttle.iopsTotalLimit=0
-s3.throttle.iopsReadLimit=0
-s3.throttle.iopsWriteLimit=0
-s3.throttle.bpsTotalMB=0
-s3.throttle.bpsReadMB=0
-s3.throttle.bpsWriteMB=0
-s3.useVirtualAddressing=false
-
-# TTL(millisecond) for distributed lock
-dlock.ttl_ms=5000
-# lock try timeout(millisecond) for distributed lock
-dlock.try_timeout_ms=300
-# lock try interval(millisecond) for distributed lock
-dlock.try_interval_ms=30
-
-#### Options for interactive with curvebs MDS
-# RPC total retry time with MDS
-bs.mds.maxRetryMs=8000
-# RPC timeout for once communication with MDS
-bs.mds.rpcTimeoutMs=500
-# The maximum timeout of RPC communicating with MDS.
-# The timeout of exponential backoff cannot exceed this value
-bs.mds.maxRPCTimeoutMs=2000
-# RPC with mds needs to sleep for a period of time before each retry
-bs.mds.rpcRetryIntervalUs=50000
-# Switch if the number of consecutive retries on the current MDS exceeds the limit.
-# The number of failures includes the number of timeout retries
-bs.mds.maxFailedTimesBeforeChangeMDS=2
-# The normal retry times for trigger wait strategy
-bs.mds.normalRetryTimesBeforeTriggerWait=3
-# sleep interval in ms for wait
-bs.mds.waitSleepMs=1000